level=info ts=2026-04-20T23:23:52.277822902Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-20T23:23:52.277846663Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-04-20T23:23:52.278200673Z caller=modules.go:1984 msg="initializing ingester query tags interceptors"
level=info ts=2026-04-20T23:23:52.279047008Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-20T23:23:52.2794793Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-20T23:23:52.279568366Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.66 inf=eth0
level=info ts=2026-04-20T23:23:52.280131786Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-4d7a1766
ts=2026-04-20T23:23:52.280199822Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-20T23:23:52.280215502Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.66 advertisePort=7946
level=debug ts=2026-04-20T23:23:52.280244053Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.66 advertisePort=7946
level=info ts=2026-04-20T23:23:52.280402207Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-20T23:23:52.284534386Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-20T23:23:52.284551535Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-20T23:23:52.284560145Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=4.160537ms
level=error ts=2026-04-20T23:23:52.284568617Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-20T23:23:52.284579404Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=error ts=2026-04-20T23:23:52.284872227Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-04-20T23:23:52.284897164Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
level=info ts=2026-04-20T23:23:52.286059065Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-04-20T23:23:52.286186005Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-20T23:23:52.286724192Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-04-20T23:23:52.28674694Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-20T23:23:52.286744638Z
level=debug ts=2026-04-20T23:23:52.286765663Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-04-20T23:23:52.286788623Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-04-20T23:23:52.286808698Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-04-20T23:23:52.286822123Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-20T23:23:52.2868148Z
level=debug ts=2026-04-20T23:23:52.286829115Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
ts=2026-04-20T23:23:52.288487648Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T23:23:52.288510125Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-20T23:23:52.289361268Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-20T23:23:52.289466946Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.130.0.66 inf=eth0
level=debug ts=2026-04-20T23:23:52.307848174Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=info ts=2026-04-20T23:23:52.307881321Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-20T23:23:52.307877199Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=info ts=2026-04-20T23:23:52.307897818Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-20T23:23:52.307886925Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-04-20T23:23:52.307913308Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-20T23:23:52.307992454Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-20T23:23:52.307941387Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=debug ts=2026-04-20T23:23:52.308296199Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-04-20T23:23:52.308299263Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-04-20T23:23:52.308305727Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-04-20T23:23:52.308723927Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=debug ts=2026-04-20T23:23:52.308726297Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-04-20T23:23:52.308736212Z caller=module_service.go:82 msg=starting module=store
level=info ts=2026-04-20T23:23:52.308739631Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-04-20T23:23:52.308771492Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-04-20T23:23:52.308812564Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-04-20T23:23:52.308818003Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-04-20T23:23:52.308818775Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=debug ts=2026-04-20T23:23:52.308823695Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=info ts=2026-04-20T23:23:52.308824007Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-20T23:23:52.308826895Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=debug ts=2026-04-20T23:23:52.308830088Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=debug ts=2026-04-20T23:23:52.308907064Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=info ts=2026-04-20T23:23:52.308913955Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-04-20T23:23:52.308930044Z caller=ingester.go:565 component=ingester msg="recovering from checkpoint"
level=info ts=2026-04-20T23:23:52.308970454Z caller=recovery.go:42 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-04-20T23:23:52.309045579Z caller=ingester.go:581 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=104.053µs errors=false
level=info ts=2026-04-20T23:23:52.309059319Z caller=ingester.go:587 component=ingester msg="recovering from WAL"
level=info ts=2026-04-20T23:23:52.309208713Z caller=ingester.go:603 component=ingester msg="WAL segment recovery finished" elapsed=280.989µs errors=false
level=info ts=2026-04-20T23:23:52.309216364Z caller=ingester.go:551 component=ingester msg="closing recoverer"
level=info ts=2026-04-20T23:23:52.309228445Z caller=ingester.go:559 component=ingester msg="WAL recovery finished" time=300.061µs
level=info ts=2026-04-20T23:23:52.309253504Z caller=wal.go:158 msg=started component=wal
level=info ts=2026-04-20T23:23:52.309336442Z caller=lifecycler.go:687 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-20T23:23:52.309340425Z caller=ingester.go:772 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=3.284140739s
level=info ts=2026-04-20T23:23:52.309361523Z caller=lifecycler.go:714 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-04-20T23:23:52.30943736Z caller=ingester.go:467 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-04-20T23:23:52.309445646Z caller=loki.go:599 msg="Loki started" startup_time=103.920374ms
level=debug ts=2026-04-20T23:23:52.309495771Z caller=lifecycler.go:552 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-04-20T23:23:52.309501992Z caller=lifecycler.go:556 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=debug ts=2026-04-20T23:23:52.309599794Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-04-20T23:23:53.817435204Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T23:23:53.817460995Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T23:23:56.067921232Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T23:23:56.067953799Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-20T23:23:57.310113443Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T23:24:02.31013049Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:24:03.896823226Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T23:24:03.896855438Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-20T23:24:07.309924251Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:24:08.292362406Z caller=logging.go:137 msg="GET /ready (503) 121.316µs"
level=warn ts=2026-04-20T23:24:12.310150924Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T23:24:17.309850134Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:24:18.290777384Z caller=logging.go:137 msg="GET /ready (503) 99.412µs"
ts=2026-04-20T23:24:18.513901578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.67:7946"
ts=2026-04-20T23:24:18.516284703Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.84:7946: dial tcp 10.129.0.84:7946: connect: connection refused"
ts=2026-04-20T23:24:18.517579806Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.83:7946: dial tcp 10.129.0.83:7946: connect: connection refused"
ts=2026-04-20T23:24:18.519123195Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.86:7946"
ts=2026-04-20T23:24:18.520962901Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.89:7946: dial tcp 10.128.0.89:7946: connect: connection refused"
ts=2026-04-20T23:24:18.521542203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.90:7946"
level=info ts=2026-04-20T23:24:18.523425694Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=26.238841514s
level=debug ts=2026-04-20T23:24:20.437414048Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T23:24:22.309682553Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-20T23:24:22.309731247Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=warn ts=2026-04-20T23:24:22.309717146Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:24:22.309736161Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:24:27.310140899Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T23:24:32.310155421Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:24:36.873309707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-20T23:24:37.310510721Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T23:24:42.310161783Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T23:24:47.31036705Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:24:47.974334469Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T23:24:47.974402739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:24:47.980201754Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:24:48.998263805Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-20T23:24:52.286339702Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:24:52.286372163Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-20T23:24:52.309678727Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:24:52.309695222Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:24:52.309723654Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:24:52.309735272Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-20T23:24:54.746481477Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:42252"
level=warn ts=2026-04-20T23:24:57.310157975Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:24:58.605799016Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:35548"
ts=2026-04-20T23:24:58.686365979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:43602"
level=warn ts=2026-04-20T23:25:02.31018746Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:25:03.100931902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:25:03.112200565Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-20T23:25:06.875794192Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-20T23:25:07.309758723Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:25:08.099451266Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-20T23:25:12.310480929Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:25:17.087543795Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T23:25:17.310156509Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T23:25:22.309610306Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:25:22.30964789Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:25:22.309668294Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:25:22.309679264Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-20T23:25:24.020548132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:25:24.028128843Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-20T23:25:24.748250161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:55000"
level=warn ts=2026-04-20T23:25:27.309948488Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:25:28.185190472Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T23:25:28.688176458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:40726"
level=warn ts=2026-04-20T23:25:32.309690249Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:25:36.878134411Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-20T23:25:37.310033898Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T23:25:42.310394614Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T23:25:47.309863694Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:25:52.286530669Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:25:52.286557284Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:25:52.309985775Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:25:52.310020057Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:25:52.31003055Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:25:52.310076415Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:25:52.659230397Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-20T23:25:54.749507159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:41030"
level=debug ts=2026-04-20T23:25:55.249274244Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-20T23:25:57.310176649Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:25:58.60983448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:49862"
level=warn ts=2026-04-20T23:26:02.309560977Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:26:06.879815093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-20T23:26:07.310316607Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T23:26:12.309720867Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:26:12.606745469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:26:12.613912702Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T23:26:17.310384428Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:26:22.309463555Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:26:22.309496214Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:26:22.309504284Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:26:22.309512337Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:26:26.641803903Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T23:26:27.310427403Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:26:28.691313951Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:41340"
level=debug ts=2026-04-20T23:26:29.475683911Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-20T23:26:29.477313149Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T23:26:32.310431036Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:26:36.881455674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-20T23:26:37.310008538Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T23:26:42.309965739Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T23:26:47.309648291Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:26:52.28651499Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:26:52.287026863Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:26:52.309443046Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:26:52.30946727Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:26:52.309474931Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:26:52.309491541Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:26:54.752344297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:40510"
level=warn ts=2026-04-20T23:26:57.309809904Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:26:57.487010672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:26:57.495995851Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:26:59.634045485Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T23:27:02.310399352Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:27:06.883011739Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-20T23:27:07.310239177Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T23:27:12.310338578Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T23:27:17.310408528Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:27:18.409024861Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-20T23:27:22.310205063Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:27:22.31023429Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:27:22.310243123Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:27:22.31026094Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:27:24.753990862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:45932"
level=debug ts=2026-04-20T23:27:26.773682317Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T23:27:27.3098178Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:27:28.694341849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:57150"
level=warn ts=2026-04-20T23:27:32.310038319Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:27:36.885072596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-20T23:27:37.309719988Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T23:27:42.3101793Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:27:45.940680443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:27:45.948846214Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T23:27:47.310303808Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:27:52.286560553Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:27:52.286571199Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:27:52.310009535Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:27:52.310034151Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-20T23:27:52.31002742Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:27:52.31004213Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T23:27:53.147854929Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-20T23:27:54.755691284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:53818"
level=warn ts=2026-04-20T23:27:57.309896798Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:27:58.61639623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:48622"
ts=2026-04-20T23:27:58.695904617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:49646"
level=warn ts=2026-04-20T23:28:02.310168299Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:28:06.887207811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-20T23:28:07.310168662Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T23:28:12.309699124Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T23:28:17.310071922Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:28:18.464921702Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-20T23:28:22.309830287Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:28:22.309858206Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-20T23:28:22.309845799Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:28:22.309867302Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T23:28:22.309916929Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T23:28:27.309544556Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:28:28.618874438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:42712"
level=warn ts=2026-04-20T23:28:32.310040848Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:28:36.888761119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-20T23:28:37.310044792Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:28:41.374144376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:28:41.382553644Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T23:28:42.310034416Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T23:28:47.310017473Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:28:47.961391008Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T23:28:52.286982383Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:28:52.287728615Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:28:52.310080528Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T23:28:52.310083748Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:28:52.310083848Z caller=checkpoint.go:612 msg="starting checkpoint"
level=debug ts=2026-04-20T23:28:52.310104267Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:28:52.310112251Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-20T23:28:52.310234107Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=debug ts=2026-04-20T23:28:52.31043826Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
level=debug ts=2026-04-20T23:28:56.665816232Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-20T23:28:57.309744858Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:28:58.69880682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:39706"
level=debug ts=2026-04-20T23:29:01.02014788Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-20T23:29:02.309492668Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:29:05.375241737Z caller=checkpoint.go:378 msg="writing series" size="237 kB"
ts=2026-04-20T23:29:06.890738946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=debug ts=2026-04-20T23:29:06.92634367Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-20T23:29:07.310416921Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:29:09.730526405Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-20T23:29:12.310415022Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:29:14.085245823Z caller=checkpoint.go:378 msg="writing series" size="494 kB"
level=debug ts=2026-04-20T23:29:17.202374548Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T23:29:17.310418973Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:29:18.439783225Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=info ts=2026-04-20T23:29:22.309431957Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:29:22.309462033Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:29:22.309470883Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:29:22.309499958Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:29:22.795366016Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
ts=2026-04-20T23:29:24.76092714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:52928"
level=debug ts=2026-04-20T23:29:27.150054298Z caller=checkpoint.go:378 msg="writing series" size="7.7 kB"
level=warn ts=2026-04-20T23:29:27.310323254Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:29:31.504315719Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-20T23:29:32.310512786Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:29:35.858905329Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
ts=2026-04-20T23:29:36.892262127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-20T23:29:37.310505364Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:29:38.32646777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:29:38.338025311Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:29:40.214420121Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=debug ts=2026-04-20T23:29:40.214446719Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=12
level=warn ts=2026-04-20T23:29:42.310127443Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:29:42.855929347Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:29:44.569641736Z caller=checkpoint.go:378 msg="writing series" size="409 kB"
level=warn ts=2026-04-20T23:29:47.310220295Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:29:48.923508785Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=debug ts=2026-04-20T23:29:52.286179753Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:29:52.287245927Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-20T23:29:52.309564118Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:29:52.309591219Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:29:52.309599117Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:29:52.309595079Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:29:53.279051978Z caller=checkpoint.go:378 msg="writing series" size="162 kB"
ts=2026-04-20T23:29:54.762458921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:58408"
level=warn ts=2026-04-20T23:29:57.310399133Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:29:57.63414307Z caller=checkpoint.go:378 msg="writing series" size="154 kB"
level=debug ts=2026-04-20T23:29:58.446569381Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T23:29:58.624261014Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:52396"
ts=2026-04-20T23:29:58.701944683Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:56878"
level=debug ts=2026-04-20T23:30:01.98835712Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=warn ts=2026-04-20T23:30:02.310293033Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:30:06.343459121Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
ts=2026-04-20T23:30:06.893405029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-20T23:30:07.309696556Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:30:10.69819691Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=warn ts=2026-04-20T23:30:12.309540309Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:30:12.766953383Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:30:15.053417287Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-20T23:30:17.309761614Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:30:19.407258261Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=info ts=2026-04-20T23:30:22.309766382Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T23:30:22.309765301Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:30:22.309807958Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:30:22.309817343Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T23:30:23.762756325Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-20T23:30:27.310261013Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:30:28.117679089Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
ts=2026-04-20T23:30:28.703930531Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:60758"
level=info ts=2026-04-20T23:30:31.949740208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:30:31.959398891Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T23:30:32.310468464Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:30:32.471822966Z caller=checkpoint.go:378 msg="writing series" size="147 kB"
level=debug ts=2026-04-20T23:30:32.471847435Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=12
level=debug ts=2026-04-20T23:30:36.827212441Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
ts=2026-04-20T23:30:36.895245342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-20T23:30:37.309688935Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:30:41.182292688Z caller=checkpoint.go:378 msg="writing series" size="92 kB"
level=warn ts=2026-04-20T23:30:42.309855167Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:30:45.445672874Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:30:45.537110163Z caller=checkpoint.go:378 msg="writing series" size="2.8 kB"
level=warn ts=2026-04-20T23:30:47.309822327Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:30:49.891894764Z caller=checkpoint.go:378 msg="writing series" size="104 kB"
level=debug ts=2026-04-20T23:30:52.2863727Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:30:52.286379611Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:30:52.287709492Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-20T23:30:52.287760407Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974141 n=1
level=debug ts=2026-04-20T23:30:52.287809267Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974141 n=1
level=info ts=2026-04-20T23:30:52.309723577Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:30:52.309754198Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:30:52.30976442Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:30:52.309818962Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:30:54.245943524Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
ts=2026-04-20T23:30:54.765251579Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:49370"
level=warn ts=2026-04-20T23:30:57.30997756Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:30:58.601743184Z caller=checkpoint.go:378 msg="writing series" size="211 kB"
level=debug ts=2026-04-20T23:30:59.957770153Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-20T23:30:59.959368895Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T23:31:02.309515748Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:31:02.955986579Z caller=checkpoint.go:378 msg="writing series" size="6.7 kB"
ts=2026-04-20T23:31:06.896992142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-20T23:31:07.309796949Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:31:07.310902563Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
level=debug ts=2026-04-20T23:31:11.669302783Z caller=checkpoint.go:378 msg="writing series" size="4.8 MB"
level=debug ts=2026-04-20T23:31:11.669329077Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.4 MB" series=9
level=warn ts=2026-04-20T23:31:12.310348971Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:31:12.639140322Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:31:16.020492878Z caller=checkpoint.go:378 msg="writing series" size="514 B"
level=warn ts=2026-04-20T23:31:17.309990442Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:31:20.375599981Z caller=checkpoint.go:378 msg="writing series" size="220 kB"
level=info ts=2026-04-20T23:31:22.309971971Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T23:31:22.309980419Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:31:22.310004631Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:31:22.310013583Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T23:31:24.730077135Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
ts=2026-04-20T23:31:24.766525658Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:49916"
level=info ts=2026-04-20T23:31:27.047062518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:31:27.057693063Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:31:27.057718781Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T23:31:27.057745097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:31:27.064691153Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T23:31:27.310440938Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:31:28.630190326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56084"
ts=2026-04-20T23:31:28.707827538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:54598"
level=debug ts=2026-04-20T23:31:29.085291221Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-20T23:31:32.310020578Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:31:33.440259817Z caller=checkpoint.go:378 msg="writing series" size="74 kB"
ts=2026-04-20T23:31:36.899269255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-20T23:31:37.309919115Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:31:37.794513881Z caller=checkpoint.go:378 msg="writing series" size="142 kB"
level=debug ts=2026-04-20T23:31:40.4991829Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-20T23:31:41.522213383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:31:41.531867689Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:31:42.149452931Z caller=checkpoint.go:378 msg="writing series" size="26 kB"
level=warn ts=2026-04-20T23:31:42.310051554Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:31:45.585923096Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:31:46.504409481Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-20T23:31:47.310418848Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:31:50.858866423Z caller=checkpoint.go:378 msg="writing series" size="809 B"
level=info ts=2026-04-20T23:31:52.287165458Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:31:52.288209621Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:31:52.309450615Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:31:52.309478341Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:31:52.309487546Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:31:52.309497213Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:31:54.767444092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:42514"
level=debug ts=2026-04-20T23:31:55.213840308Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-20T23:31:57.310233752Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:31:58.710045093Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:36782"
level=debug ts=2026-04-20T23:31:59.569422675Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=warn ts=2026-04-20T23:32:02.309838751Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:32:03.217343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:32:03.231845114Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:32:03.924128417Z caller=checkpoint.go:378 msg="writing series" size="292 kB"
ts=2026-04-20T23:32:06.901267239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-20T23:32:07.310401954Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:32:08.278651671Z caller=checkpoint.go:378 msg="writing series" size="219 kB"
level=debug ts=2026-04-20T23:32:08.27867809Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=13
level=debug ts=2026-04-20T23:32:12.001258307Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T23:32:12.310379189Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:32:12.633884256Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=debug ts=2026-04-20T23:32:16.988733615Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-20T23:32:17.310163435Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:32:21.343425705Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-04-20T23:32:22.310382354Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:32:22.31041169Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:32:22.310428449Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:32:22.310431794Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:32:25.698323496Z caller=checkpoint.go:378 msg="writing series" size="296 kB"
level=warn ts=2026-04-20T23:32:27.309962016Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:32:28.633671815Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:45182"
level=debug ts=2026-04-20T23:32:30.053086863Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=warn ts=2026-04-20T23:32:32.310446192Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:32:34.40751092Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=debug ts=2026-04-20T23:32:34.70696582Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T23:32:36.902692792Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-20T23:32:37.310271098Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:32:38.762567214Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=debug ts=2026-04-20T23:32:42.154216839Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T23:32:42.309670614Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:32:43.117699159Z caller=checkpoint.go:378 msg="writing series" size="135 kB"
level=warn ts=2026-04-20T23:32:47.309700566Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:32:47.472085316Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=debug ts=2026-04-20T23:32:51.828321817Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-20T23:32:51.828347901Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=10
level=info ts=2026-04-20T23:32:52.28690842Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:32:52.288016813Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:32:52.310369611Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:32:52.310396382Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:32:52.310404788Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:32:52.310387878Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:32:52.554700846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:32:52.56295473Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:32:56.1822827Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=warn ts=2026-04-20T23:32:57.309828454Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:32:58.63592464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:44254"
level=debug ts=2026-04-20T23:33:00.538724458Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=warn ts=2026-04-20T23:33:02.310003508Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:33:04.891991572Z caller=checkpoint.go:378 msg="writing series" size="273 kB"
ts=2026-04-20T23:33:06.904162717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-20T23:33:07.30953527Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:33:09.246775006Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-20T23:33:12.309811875Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:33:13.600807051Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=debug ts=2026-04-20T23:33:16.680131137Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T23:33:17.310243845Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:33:17.360950638Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T23:33:17.956382369Z caller=checkpoint.go:378 msg="writing series" size="183 kB"
level=info ts=2026-04-20T23:33:22.309931532Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:33:22.309963075Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:33:22.309972107Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:33:22.309979267Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:33:22.311037882Z caller=checkpoint.go:390 msg="flushing series" totalSize="701 kB" series=6
level=info ts=2026-04-20T23:33:22.316679374Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-04-20T23:33:22.320113931Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.009729962s
level=warn ts=2026-04-20T23:33:27.309884194Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:33:28.637854015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:50688"
level=warn ts=2026-04-20T23:33:32.309513149Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:33:36.738256591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:33:36.751438252Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-20T23:33:36.906652782Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-20T23:33:37.31023544Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T23:33:42.309664786Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T23:33:47.309916261Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:33:49.969343958Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T23:33:52.286899452Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:33:52.287920919Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:33:52.310259652Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:33:52.310287332Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-20T23:33:52.310275285Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:33:52.310295775Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-20T23:33:52.310299108Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-20T23:33:52.31051298Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=debug ts=2026-04-20T23:33:52.310721202Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
ts=2026-04-20T23:33:54.774485826Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:52818"
level=debug ts=2026-04-20T23:33:56.666450441Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-20T23:33:57.310289479Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:33:58.716873231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:59978"
level=debug ts=2026-04-20T23:34:01.021411036Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-20T23:34:02.310076071Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:34:05.375803186Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
ts=2026-04-20T23:34:06.908495529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-20T23:34:07.310644495Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:34:09.730669653Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-20T23:34:12.309868437Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:34:13.065549153Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T23:34:14.085914367Z caller=checkpoint.go:378 msg="writing series" size="451 kB"
level=warn ts=2026-04-20T23:34:17.309955715Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:34:18.439978069Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
level=debug ts=2026-04-20T23:34:20.584121524Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T23:34:22.309844355Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T23:34:22.309851733Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:34:22.309882748Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:34:22.309890816Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T23:34:22.795700116Z caller=checkpoint.go:378 msg="writing series" size="113 kB"
level=debug ts=2026-04-20T23:34:27.149797448Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-20T23:34:27.310184048Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:34:28.641223231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:36086"
level=debug ts=2026-04-20T23:34:31.505009843Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-20T23:34:32.310197341Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:34:34.461774937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:34:34.471765404Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:34:35.85994422Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
ts=2026-04-20T23:34:36.91006516Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-20T23:34:37.309744738Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:34:40.214390545Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=warn ts=2026-04-20T23:34:42.309649814Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:34:44.569865129Z caller=checkpoint.go:378 msg="writing series" size="409 kB"
level=debug ts=2026-04-20T23:34:44.569889428Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=13
level=warn ts=2026-04-20T23:34:47.310199463Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:34:48.923710082Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=debug ts=2026-04-20T23:34:51.226316156Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T23:34:52.286466506Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:34:52.287298842Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:34:52.309564005Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:34:52.309584279Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:34:52.30959302Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:34:52.309617276Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:34:53.279468989Z caller=checkpoint.go:378 msg="writing series" size="301 kB"
level=warn ts=2026-04-20T23:34:57.310063823Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:34:57.633435822Z caller=checkpoint.go:378 msg="writing series" size="183 kB"
level=debug ts=2026-04-20T23:35:01.988524712Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=warn ts=2026-04-20T23:35:02.310293955Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:35:02.574297487Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T23:35:02.575936636Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-20T23:35:06.343794456Z caller=checkpoint.go:378 msg="writing series" size="92 kB"
ts=2026-04-20T23:35:06.912191225Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-20T23:35:07.30988576Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:35:10.698662422Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=warn ts=2026-04-20T23:35:12.310500273Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:35:14.830854547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:35:14.838947494Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:35:15.052715757Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-20T23:35:17.310587745Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:35:19.40797496Z caller=checkpoint.go:378 msg="writing series" size="4.4 kB"
level=debug ts=2026-04-20T23:35:20.32286862Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T23:35:22.309835916Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:35:22.309864566Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:35:22.309881582Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:35:22.30990012Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:35:23.762903311Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
ts=2026-04-20T23:35:24.779196913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:41190"
level=warn ts=2026-04-20T23:35:27.309851038Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:35:28.118234496Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
ts=2026-04-20T23:35:28.644864696Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:49330"
level=warn ts=2026-04-20T23:35:32.310375255Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:35:32.472743199Z caller=checkpoint.go:378 msg="writing series" size="174 kB"
level=debug ts=2026-04-20T23:35:36.827103271Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
ts=2026-04-20T23:35:36.913695987Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-20T23:35:37.309741626Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:35:41.182564567Z caller=checkpoint.go:378 msg="writing series" size="92 kB"
level=warn ts=2026-04-20T23:35:42.309494552Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:35:45.537487628Z caller=checkpoint.go:378 msg="writing series" size="2.8 kB"
level=warn ts=2026-04-20T23:35:47.310201356Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:35:49.892145969Z caller=checkpoint.go:378 msg="writing series" size="136 kB"
level=debug ts=2026-04-20T23:35:49.89217047Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=15
level=debug ts=2026-04-20T23:35:51.695276696Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-20T23:35:52.286380222Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:35:52.286713291Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:35:52.31014334Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:35:52.310177313Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:35:52.310197932Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:35:52.310220378Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:35:54.246481004Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=debug ts=2026-04-20T23:35:54.351089855Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-20T23:35:54.780244983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:43030"
level=warn ts=2026-04-20T23:35:57.310199638Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:35:57.901434849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:35:57.909833725Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:35:58.601258348Z caller=checkpoint.go:378 msg="writing series" size="211 kB"
level=warn ts=2026-04-20T23:36:02.309923891Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:36:02.956348486Z caller=checkpoint.go:378 msg="writing series" size="6.7 kB"
ts=2026-04-20T23:36:06.915299009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-20T23:36:07.309838249Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:36:07.310953153Z caller=checkpoint.go:378 msg="writing series" size="217 kB"
level=debug ts=2026-04-20T23:36:11.669733907Z caller=checkpoint.go:378 msg="writing series" size="5.4 MB"
level=debug ts=2026-04-20T23:36:11.669759706Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.8 MB" series=5
level=warn ts=2026-04-20T23:36:12.310032229Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:36:16.021311848Z caller=checkpoint.go:378 msg="writing series" size="514 B"
level=warn ts=2026-04-20T23:36:17.309599375Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:36:20.376442602Z caller=checkpoint.go:378 msg="writing series" size="231 kB"
level=info ts=2026-04-20T23:36:22.310224424Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:36:22.31026244Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:36:22.310273285Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:36:22.310282014Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:36:22.365489864Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:36:24.730850589Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
ts=2026-04-20T23:36:24.781799984Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:57960"
level=debug ts=2026-04-20T23:36:27.127570207Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-20T23:36:27.309522701Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:36:29.085647625Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-20T23:36:32.309553429Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:36:33.440241078Z caller=checkpoint.go:378 msg="writing series" size="80 kB"
ts=2026-04-20T23:36:36.917236007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-20T23:36:37.309997114Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:36:37.795575723Z caller=checkpoint.go:378 msg="writing series" size="142 kB"
level=debug ts=2026-04-20T23:36:42.149776613Z caller=checkpoint.go:378 msg="writing series" size="28 kB"
level=warn ts=2026-04-20T23:36:42.309687218Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:36:46.504421019Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-20T23:36:47.3103799Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:36:50.859871654Z caller=checkpoint.go:378 msg="writing series" size="809 B"
level=info ts=2026-04-20T23:36:52.286972304Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:36:52.287515093Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:36:52.309785837Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:36:52.309816819Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-20T23:36:52.309812903Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:36:52.309825317Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-20T23:36:53.140613769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:36:53.147935654Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:36:54.659347385Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:36:55.21501707Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-20T23:36:57.309786757Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:36:59.569427378Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
level=warn ts=2026-04-20T23:37:02.309900679Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:37:02.982275053Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T23:37:03.924067542Z caller=checkpoint.go:378 msg="writing series" size="160 kB"
ts=2026-04-20T23:37:06.919077923Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-20T23:37:07.309671673Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:37:08.279554758Z caller=checkpoint.go:378 msg="writing series" size="222 kB"
level=warn ts=2026-04-20T23:37:12.310470042Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:37:12.634075628Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=debug ts=2026-04-20T23:37:16.988149408Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-20T23:37:17.309734776Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:37:21.343245935Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=debug ts=2026-04-20T23:37:21.865861628Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T23:37:22.310113981Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:37:22.310147903Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:37:22.310156986Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:37:22.310173921Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:37:25.699015628Z caller=checkpoint.go:378 msg="writing series" size="296 kB"
level=debug ts=2026-04-20T23:37:25.699045305Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=17
level=warn ts=2026-04-20T23:37:27.310299816Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:37:30.053584268Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=warn ts=2026-04-20T23:37:32.310532016Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:37:34.408547778Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
ts=2026-04-20T23:37:36.920403701Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-20T23:37:37.310568426Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:37:38.762947534Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-20T23:37:42.31006786Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:37:43.11750387Z caller=checkpoint.go:378 msg="writing series" size="191 kB"
level=info ts=2026-04-20T23:37:45.889850901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:37:45.897992293Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:37:45.898018995Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T23:37:45.898046285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:37:45.906549901Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T23:37:47.309761077Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:37:47.47225592Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=debug ts=2026-04-20T23:37:48.709984267Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:37:51.82885643Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB"
level=debug ts=2026-04-20T23:37:51.828880147Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=6
level=info ts=2026-04-20T23:37:52.286559735Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:37:52.28688388Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:37:52.310330038Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:37:52.310356322Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:37:52.310364385Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:37:52.310380636Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:37:56.181967316Z caller=checkpoint.go:378 msg="writing series" size="302 kB"
level=warn ts=2026-04-20T23:37:57.30985479Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:37:58.524480204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:37:58.533392005Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-20T23:37:58.729405064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:32816"
level=debug ts=2026-04-20T23:38:00.536560229Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=warn ts=2026-04-20T23:38:02.310204444Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:38:04.508920883Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T23:38:04.891993762Z caller=checkpoint.go:378 msg="writing series" size="292 kB"
ts=2026-04-20T23:38:06.922353223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-20T23:38:07.310612463Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:38:09.247151964Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-20T23:38:12.310113967Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:38:13.601383724Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=warn ts=2026-04-20T23:38:17.309701367Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:38:17.956766858Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=debug ts=2026-04-20T23:38:19.494826886Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T23:38:22.309607913Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:38:22.309659084Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-20T23:38:22.309656904Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:38:22.309670837Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T23:38:22.31171302Z caller=checkpoint.go:390 msg="flushing series" totalSize="840 kB" series=6
level=info ts=2026-04-20T23:38:22.319508995Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-04-20T23:38:22.320834518Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.010166429s
ts=2026-04-20T23:38:24.789528943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:33750"
level=info ts=2026-04-20T23:38:24.797121614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:38:24.807718557Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T23:38:27.309693791Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:38:28.731532067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:45548"
level=warn ts=2026-04-20T23:38:32.310180601Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:38:36.924842274Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-20T23:38:37.310402508Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T23:38:42.310282231Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T23:38:47.309645205Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:38:49.838944266Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:38:51.716480945Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T23:38:51.718091679Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-20T23:38:52.286398983Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:38:52.28640344Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=warn ts=2026-04-20T23:38:52.309817783Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:38:52.309823584Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-20T23:38:52.309859693Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:38:52.309885045Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:38:52.309895077Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-20T23:38:52.309989898Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=debug ts=2026-04-20T23:38:52.310210051Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
level=debug ts=2026-04-20T23:38:56.665278322Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-20T23:38:57.309958807Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:38:58.73318124Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:47564"
level=debug ts=2026-04-20T23:39:01.020193873Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-20T23:39:02.309571578Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:39:05.375489921Z caller=checkpoint.go:378 msg="writing series" size="140 kB"
ts=2026-04-20T23:39:06.927079875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-20T23:39:07.309772238Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:39:09.729635894Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-20T23:39:12.310382722Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:39:12.487652082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:39:12.497236638Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:39:14.085460637Z caller=checkpoint.go:378 msg="writing series" size="557 kB"
level=warn ts=2026-04-20T23:39:17.310077488Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:39:18.439537786Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=info ts=2026-04-20T23:39:22.309836416Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T23:39:22.309846788Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:39:22.309866924Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:39:22.309876153Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T23:39:22.794459945Z caller=checkpoint.go:378 msg="writing series" size="167 kB"
level=debug ts=2026-04-20T23:39:25.264734159Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:39:27.149725438Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
level=warn ts=2026-04-20T23:39:27.310291277Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:39:28.659150544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:34686"
level=debug ts=2026-04-20T23:39:31.503866109Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=warn ts=2026-04-20T23:39:32.309996712Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:39:35.85902973Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=debug ts=2026-04-20T23:39:35.859051837Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
ts=2026-04-20T23:39:36.928860507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-20T23:39:37.309956574Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:39:40.214386408Z caller=checkpoint.go:378 msg="writing series" size="55 kB"
level=debug ts=2026-04-20T23:39:40.351525419Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-20T23:39:42.310288917Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:39:44.569953695Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-04-20T23:39:44.569978692Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=2
level=warn ts=2026-04-20T23:39:47.31003587Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:39:48.923661228Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=info ts=2026-04-20T23:39:52.286736508Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:39:52.287178823Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:39:52.309452864Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:39:52.309478972Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:39:52.309503658Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:39:52.309499713Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:39:53.278453008Z caller=checkpoint.go:378 msg="writing series" size="399 kB"
level=debug ts=2026-04-20T23:39:56.35727986Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T23:39:57.309845543Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:39:57.633545692Z caller=checkpoint.go:378 msg="writing series" size="216 kB"
ts=2026-04-20T23:39:58.661717262Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:53912"
level=debug ts=2026-04-20T23:40:01.988707281Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=warn ts=2026-04-20T23:40:02.309657601Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:40:06.343235335Z caller=checkpoint.go:378 msg="writing series" size="92 kB"
ts=2026-04-20T23:40:06.930533868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-20T23:40:07.310511464Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:40:10.69754023Z caller=checkpoint.go:378 msg="writing series" size="105 kB"
level=info ts=2026-04-20T23:40:12.274449259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:40:12.287135163Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T23:40:12.310528432Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:40:15.053026386Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-20T23:40:17.310482218Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:40:19.407185114Z caller=checkpoint.go:378 msg="writing series" size="5.6 kB"
level=info ts=2026-04-20T23:40:22.310256133Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:40:22.310297533Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:40:22.310312156Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:40:22.310308082Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:40:23.149408402Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T23:40:23.762769332Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-20T23:40:27.309727922Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:40:28.117274478Z caller=checkpoint.go:378 msg="writing series" size="73 kB"
level=debug ts=2026-04-20T23:40:30.935988379Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T23:40:30.936069179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:40:30.943213508Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-20T23:40:32.310236471Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:40:32.472603574Z caller=checkpoint.go:378 msg="writing series" size="162 kB"
level=debug ts=2026-04-20T23:40:32.472645357Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=debug ts=2026-04-20T23:40:36.827347544Z caller=checkpoint.go:378 msg="writing series" size="183 kB"
ts=2026-04-20T23:40:36.9319576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-20T23:40:37.310306795Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:40:41.181702029Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=warn ts=2026-04-20T23:40:42.30989281Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:40:45.536159004Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=info ts=2026-04-20T23:40:46.069659218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:40:46.079747592Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-20T23:40:47.310417281Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:40:49.891902218Z caller=checkpoint.go:378 msg="writing series" size="198 kB"
level=info ts=2026-04-20T23:40:52.286503435Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:40:52.287014549Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:40:52.310452378Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:40:52.310480213Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-20T23:40:52.310468649Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:40:52.310489064Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-20T23:40:53.526551916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:40:53.535386281Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:40:54.246241441Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
ts=2026-04-20T23:40:54.799388661Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:49146"
level=debug ts=2026-04-20T23:40:56.948396275Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T23:40:57.310475118Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:40:58.601112707Z caller=checkpoint.go:378 msg="writing series" size="89 kB"
level=warn ts=2026-04-20T23:41:02.310342422Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:41:02.955883046Z caller=checkpoint.go:378 msg="writing series" size="6.7 kB"
ts=2026-04-20T23:41:06.932745757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-20T23:41:07.310213515Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:41:07.310282985Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=debug ts=2026-04-20T23:41:11.669724657Z caller=checkpoint.go:378 msg="writing series" size="5.5 MB"
level=debug ts=2026-04-20T23:41:11.66975078Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.3 MB" series=9
level=warn ts=2026-04-20T23:41:12.309578516Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:41:15.213904824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:41:15.22403819Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-20T23:41:16.02036702Z caller=checkpoint.go:378 msg="writing series" size="514 B"
level=warn ts=2026-04-20T23:41:17.309568392Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:41:18.533174689Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T23:41:20.376140416Z caller=checkpoint.go:378 msg="writing series" size="360 kB"
level=warn ts=2026-04-20T23:41:22.309592227Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:41:22.309591726Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:41:22.309638475Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:41:22.309647688Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T23:41:24.729587803Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=warn ts=2026-04-20T23:41:27.310422128Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:41:29.085200648Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=debug ts=2026-04-20T23:41:29.971835517Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T23:41:32.31033998Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:41:33.440294681Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
ts=2026-04-20T23:41:36.934155001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-20T23:41:37.309819076Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:41:37.794343821Z caller=checkpoint.go:378 msg="writing series" size="142 kB"
level=info ts=2026-04-20T23:41:40.222696584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:41:40.231759854Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:41:42.149533371Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=warn ts=2026-04-20T23:41:42.310131974Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:41:46.504800132Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-20T23:41:47.30974277Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:41:50.859436521Z caller=checkpoint.go:378 msg="writing series" size="809 B"
level=info ts=2026-04-20T23:41:52.286821485Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:41:52.2877503Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:41:52.310070918Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:41:52.310100998Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:41:52.310110973Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:41:52.310120715Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:41:54.802494094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:40550"
level=debug ts=2026-04-20T23:41:55.214138031Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=debug ts=2026-04-20T23:41:55.822695432Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T23:41:57.310047817Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:41:59.568475297Z caller=checkpoint.go:378 msg="writing series" size="109 kB"
level=info ts=2026-04-20T23:42:00.209074097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:42:00.218684004Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-20T23:42:02.310249288Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:42:03.923880414Z caller=checkpoint.go:378 msg="writing series" size="345 kB"
level=debug ts=2026-04-20T23:42:03.923904193Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=12
ts=2026-04-20T23:42:06.935733923Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-20T23:42:07.309840887Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:42:08.278197732Z caller=checkpoint.go:378 msg="writing series" size="166 kB"
level=debug ts=2026-04-20T23:42:09.807804886Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-20T23:42:12.309821896Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:42:12.633376732Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=debug ts=2026-04-20T23:42:16.987719474Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-20T23:42:17.310384473Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:42:21.342700442Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-04-20T23:42:22.309959079Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:42:22.309991125Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:42:22.310000728Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:42:22.310020056Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:42:24.804797505Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:42:25.697931459Z caller=checkpoint.go:378 msg="writing series" size="134 kB"
level=warn ts=2026-04-20T23:42:27.310488396Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:42:30.052521735Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=warn ts=2026-04-20T23:42:32.31043703Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:42:34.407207292Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
ts=2026-04-20T23:42:36.937156062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-20T23:42:37.310310668Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:42:38.762698505Z caller=checkpoint.go:378 msg="writing series" size="77 kB"
level=info ts=2026-04-20T23:42:39.18001719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:42:39.190954534Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T23:42:42.310071567Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:42:43.117687554Z caller=checkpoint.go:378 msg="writing series" size="248 kB"
level=warn ts=2026-04-20T23:42:47.309531762Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:42:47.471907067Z caller=checkpoint.go:378 msg="writing series" size="84 kB"
level=debug ts=2026-04-20T23:42:49.598740932Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:42:51.828069176Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB"
level=debug ts=2026-04-20T23:42:51.828099002Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.2 MB" series=11
level=info ts=2026-04-20T23:42:52.286958495Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:42:52.287893354Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:42:52.310244079Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:42:52.31027579Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:42:52.310287214Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:42:52.310247194Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:42:54.805680757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:33072"
level=debug ts=2026-04-20T23:42:56.182271609Z caller=checkpoint.go:378 msg="writing series" size="192 kB"
level=warn ts=2026-04-20T23:42:57.310233964Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:42:58.275740897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:42:58.28264174Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-20T23:42:58.6715078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:50938"
ts=2026-04-20T23:42:58.747836763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:37298"
level=debug ts=2026-04-20T23:42:58.77761889Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T23:42:58.779438337Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-20T23:43:00.536731862Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-20T23:43:02.3097329Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:43:04.89159435Z caller=checkpoint.go:378 msg="writing series" size="390 kB"
ts=2026-04-20T23:43:06.938450332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-20T23:43:07.30993635Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:43:09.245954451Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-20T23:43:12.309954263Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:43:13.601358797Z caller=checkpoint.go:378 msg="writing series" size="127 kB"
level=debug ts=2026-04-20T23:43:15.876814193Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T23:43:17.310609881Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:43:17.955792376Z caller=checkpoint.go:378 msg="writing series" size="185 kB"
level=info ts=2026-04-20T23:43:22.310000022Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:43:22.310028753Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:43:22.310036843Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:43:22.310047074Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:43:22.311080141Z caller=checkpoint.go:390 msg="flushing series" totalSize="899 kB" series=6
level=info ts=2026-04-20T23:43:22.314071684Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-04-20T23:43:22.31552376Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.005375765s
level=warn ts=2026-04-20T23:43:27.30989897Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:43:28.673796773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:60580"
ts=2026-04-20T23:43:28.750391593Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:46558"
level=info ts=2026-04-20T23:43:29.683873281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:43:29.69261971Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T23:43:32.309615143Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:43:34.421156268Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T23:43:36.939906293Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-20T23:43:37.309690435Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:43:39.89087669Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T23:43:42.310486104Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T23:43:47.309888552Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:43:49.562517534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:43:49.570694297Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-20T23:43:52.286412316Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:43:52.286674867Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:43:52.310130091Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-20T23:43:52.310154026Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:43:52.31018566Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:43:52.310195654Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:43:52.310189858Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:43:52.310327819Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=debug ts=2026-04-20T23:43:52.310550525Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
level=debug ts=2026-04-20T23:43:56.113534287Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-20T23:43:57.310346456Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:43:58.676305805Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:42090"
level=debug ts=2026-04-20T23:43:59.916971857Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-20T23:44:02.310167989Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:44:03.720078974Z caller=checkpoint.go:378 msg="writing series" size="198 kB"
ts=2026-04-20T23:44:06.941733329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-20T23:44:07.309688042Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:44:07.522828608Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=debug ts=2026-04-20T23:44:08.408849319Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:44:11.325927463Z caller=checkpoint.go:378 msg="writing series" size="624 kB"
level=warn ts=2026-04-20T23:44:12.309802431Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:44:15.128278207Z caller=checkpoint.go:378 msg="writing series" size="328 kB"
level=debug ts=2026-04-20T23:44:15.128311181Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=7
level=warn ts=2026-04-20T23:44:17.310354238Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:44:18.931089918Z caller=checkpoint.go:378 msg="writing series" size="307 kB"
level=info ts=2026-04-20T23:44:22.309433128Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:44:22.309463686Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:44:22.309472355Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:44:22.309510002Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:44:22.733802433Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=info ts=2026-04-20T23:44:26.013272576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:44:26.021223703Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:44:26.021256359Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T23:44:26.021284533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:44:26.028937287Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:44:26.536538262Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=warn ts=2026-04-20T23:44:27.30975097Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:44:28.051220446Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T23:44:28.67821854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:54274"
ts=2026-04-20T23:44:28.754177776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:58480"
level=debug ts=2026-04-20T23:44:30.339394617Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=warn ts=2026-04-20T23:44:32.30989404Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:44:34.142457882Z caller=checkpoint.go:378 msg="writing series" size="63 kB"
ts=2026-04-20T23:44:36.943958524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-20T23:44:37.309534454Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:44:37.954199883Z caller=checkpoint.go:378 msg="writing series" size="7.5 MB"
level=debug ts=2026-04-20T23:44:37.954237089Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.1 MB" series=6
level=debug ts=2026-04-20T23:44:38.397024459Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:44:41.74749307Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=warn ts=2026-04-20T23:44:42.310384224Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:44:43.449771254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:44:43.459008581Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:44:45.550510603Z caller=checkpoint.go:378 msg="writing series" size="335 kB"
level=warn ts=2026-04-20T23:44:47.309895143Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:44:47.640707218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:44:47.649580268Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-20T23:44:49.353024992Z caller=checkpoint.go:378 msg="writing series" size="246 kB"
level=info ts=2026-04-20T23:44:52.286570023Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:44:52.286579097Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:44:52.31007928Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T23:44:52.31009172Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:44:52.310115587Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:44:52.310124875Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T23:44:53.155728482Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
ts=2026-04-20T23:44:54.812924505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:37484"
level=debug ts=2026-04-20T23:44:56.959466515Z caller=checkpoint.go:378 msg="writing series" size="76 kB"
level=warn ts=2026-04-20T23:44:57.310436172Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:44:58.755757454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:39876"
level=debug ts=2026-04-20T23:45:00.761781605Z caller=checkpoint.go:378 msg="writing series" size="158 kB"
level=warn ts=2026-04-20T23:45:02.310299301Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:45:04.564956431Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
ts=2026-04-20T23:45:06.945487681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-20T23:45:07.310086778Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:45:08.367205777Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=debug ts=2026-04-20T23:45:10.49783495Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T23:45:12.170511326Z caller=checkpoint.go:378 msg="writing series" size="6.5 kB"
level=warn ts=2026-04-20T23:45:12.310423295Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:45:13.089242976Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T23:45:14.206058562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:45:14.215703574Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:45:15.973495813Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-20T23:45:17.31052036Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:45:19.776364375Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
level=info ts=2026-04-20T23:45:22.309672279Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:45:22.309713475Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:45:22.309725529Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:45:22.309734015Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:45:23.578658615Z caller=checkpoint.go:378 msg="writing series" size="188 kB"
level=debug ts=2026-04-20T23:45:23.578687278Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=12
level=warn ts=2026-04-20T23:45:27.310184274Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:45:27.3816826Z caller=checkpoint.go:378 msg="writing series" size="300 kB"
ts=2026-04-20T23:45:28.682501621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:46734"
level=debug ts=2026-04-20T23:45:31.183826853Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=warn ts=2026-04-20T23:45:32.30999702Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:45:34.986937909Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
ts=2026-04-20T23:45:36.94673972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-20T23:45:37.310000992Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:45:38.790221729Z caller=checkpoint.go:378 msg="writing series" size="250 kB"
level=info ts=2026-04-20T23:45:40.908006314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:45:40.921191877Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-20T23:45:42.309966001Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:45:42.592846952Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=debug ts=2026-04-20T23:45:46.395571959Z caller=checkpoint.go:378 msg="writing series" size="26 kB"
level=warn ts=2026-04-20T23:45:47.310309488Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:45:47.37893123Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T23:45:48.269439413Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:45:50.198342263Z caller=checkpoint.go:378 msg="writing series" size="80 kB"
level=info ts=2026-04-20T23:45:52.286374226Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:45:52.286799179Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=debug ts=2026-04-20T23:45:52.288087373Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-20T23:45:52.288137494Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974142 n=1
level=debug ts=2026-04-20T23:45:52.288187501Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974142 n=1
level=info ts=2026-04-20T23:45:52.310221399Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:45:52.310251259Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:45:52.31026302Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:45:52.310286367Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:45:54.000891801Z caller=checkpoint.go:378 msg="writing series" size="6.7 kB"
ts=2026-04-20T23:45:54.816479075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:47474"
level=warn ts=2026-04-20T23:45:57.310138237Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:45:57.80405491Z caller=checkpoint.go:378 msg="writing series" size="250 kB"
ts=2026-04-20T23:45:58.759613602Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:40846"
level=debug ts=2026-04-20T23:46:01.610079838Z caller=checkpoint.go:378 msg="writing series" size="6.0 MB"
level=debug ts=2026-04-20T23:46:01.610104507Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.0 MB" series=10
level=warn ts=2026-04-20T23:46:02.310333921Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:46:05.40935016Z caller=checkpoint.go:378 msg="writing series" size="514 B"
ts=2026-04-20T23:46:06.948158412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-20T23:46:07.310468598Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:46:08.356681502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:46:08.36642553Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:46:09.212361099Z caller=checkpoint.go:378 msg="writing series" size="398 kB"
level=warn ts=2026-04-20T23:46:12.310178652Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:46:13.018755553Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=debug ts=2026-04-20T23:46:16.818781759Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-20T23:46:17.309916529Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:46:18.297733416Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:46:20.620593281Z caller=checkpoint.go:378 msg="writing series" size="93 kB"
level=info ts=2026-04-20T23:46:22.309484711Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T23:46:22.309507109Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:46:22.309531807Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:46:22.309540489Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-20T23:46:22.42479014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:46:22.437955943Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-20T23:46:24.423837766Z caller=checkpoint.go:378 msg="writing series" size="142 kB"
level=warn ts=2026-04-20T23:46:27.310123967Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:46:28.226844296Z caller=checkpoint.go:378 msg="writing series" size="30 kB"
level=debug ts=2026-04-20T23:46:32.02961994Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-20T23:46:32.309612624Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:46:35.832220511Z caller=checkpoint.go:378 msg="writing series" size="809 B"
ts=2026-04-20T23:46:36.950107366Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-20T23:46:37.309752441Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:46:39.635541302Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=warn ts=2026-04-20T23:46:42.309534053Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:46:43.438302522Z caller=checkpoint.go:378 msg="writing series" size="225 kB"
level=debug ts=2026-04-20T23:46:46.946491616Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T23:46:46.948206185Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-20T23:46:47.240287118Z caller=checkpoint.go:378 msg="writing series" size="131 kB"
level=debug ts=2026-04-20T23:46:47.24031226Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=12
level=warn ts=2026-04-20T23:46:47.31035114Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:46:47.561745546Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:46:51.043946827Z caller=checkpoint.go:378 msg="writing series" size="214 kB"
level=debug ts=2026-04-20T23:46:52.286199587Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:46:52.287254848Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-20T23:46:52.309633724Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T23:46:52.309616026Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:46:52.309662465Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:46:52.309674196Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-20T23:46:54.825074822Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:43636"
level=debug ts=2026-04-20T23:46:54.846928832Z caller=checkpoint.go:378 msg="writing series" size="411 kB"
level=info ts=2026-04-20T23:46:55.257273954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:46:55.264048207Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T23:46:57.310207072Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:46:58.648911569Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=warn ts=2026-04-20T23:47:02.310236834Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:47:02.451493642Z caller=checkpoint.go:378 msg="writing series" size="9.9 kB"
level=debug ts=2026-04-20T23:47:06.254764152Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
ts=2026-04-20T23:47:06.951587816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-20T23:47:07.310428704Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:47:10.057879762Z caller=checkpoint.go:378 msg="writing series" size="208 kB"
level=info ts=2026-04-20T23:47:11.088502777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:47:11.097062775Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:47:11.097088241Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T23:47:11.097117899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:47:11.104499016Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-20T23:47:12.309505219Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:47:13.859950408Z caller=checkpoint.go:378 msg="writing series" size="63 kB"
level=warn ts=2026-04-20T23:47:17.310478982Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:47:17.663019398Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=debug ts=2026-04-20T23:47:18.857468841Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:47:21.465557541Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=info ts=2026-04-20T23:47:22.309807757Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:47:22.309842821Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:47:22.309851538Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:47:22.309903279Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:47:25.268920944Z caller=checkpoint.go:378 msg="writing series" size="77 kB"
level=debug ts=2026-04-20T23:47:25.268945033Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-04-20T23:47:26.312127815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:47:26.324359593Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-20T23:47:27.309540706Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:47:28.689863805Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:49132"
level=debug ts=2026-04-20T23:47:29.071980786Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
level=debug ts=2026-04-20T23:47:29.955954924Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-20T23:47:32.309794046Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:47:32.874238955Z caller=checkpoint.go:378 msg="writing series" size="8.2 kB"
level=debug ts=2026-04-20T23:47:36.677506134Z caller=checkpoint.go:378 msg="writing series" size="84 kB"
ts=2026-04-20T23:47:36.953514967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-20T23:47:37.310526784Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:47:40.480581285Z caller=checkpoint.go:378 msg="writing series" size="1.8 MB"
level=debug ts=2026-04-20T23:47:40.480605566Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=4
level=warn ts=2026-04-20T23:47:42.309694714Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:47:44.283262475Z caller=checkpoint.go:378 msg="writing series" size="309 kB"
level=info ts=2026-04-20T23:47:44.686734612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:47:44.694467504Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T23:47:47.309856259Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:47:48.086076535Z caller=checkpoint.go:378 msg="writing series" size="186 kB"
level=debug ts=2026-04-20T23:47:51.111248749Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:47:51.888526498Z caller=checkpoint.go:378 msg="writing series" size="2.3 kB"
level=info ts=2026-04-20T23:47:52.286370544Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:47:52.286953325Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:47:52.310394004Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T23:47:52.3104094Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:47:52.31043096Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:47:52.310442059Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T23:47:55.691745967Z caller=checkpoint.go:378 msg="writing series" size="102 kB"
level=warn ts=2026-04-20T23:47:57.309747912Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:47:59.494583159Z caller=checkpoint.go:378 msg="writing series" size="307 kB"
level=info ts=2026-04-20T23:48:02.002582518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:48:02.010911266Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-20T23:48:02.309698397Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:48:03.297196187Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
ts=2026-04-20T23:48:06.955345841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=debug ts=2026-04-20T23:48:07.099966955Z caller=checkpoint.go:378 msg="writing series" size="399 kB"
level=debug ts=2026-04-20T23:48:07.099996031Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=7
level=warn ts=2026-04-20T23:48:07.310260973Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:48:10.903003558Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=debug ts=2026-04-20T23:48:11.387362652Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-20T23:48:12.310499056Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:48:14.705520862Z caller=checkpoint.go:378 msg="writing series" size="185 kB"
level=warn ts=2026-04-20T23:48:17.309771839Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:48:18.508641987Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=debug ts=2026-04-20T23:48:20.69174933Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T23:48:22.309663878Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:48:22.309673053Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:48:22.309698861Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:48:22.309707648Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T23:48:22.310770347Z caller=checkpoint.go:390 msg="flushing series" totalSize="224 kB" series=3
level=info ts=2026-04-20T23:48:22.313656497Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-04-20T23:48:22.317212043Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.0067253s
level=warn ts=2026-04-20T23:48:27.309797038Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T23:48:32.310487669Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:48:36.956931208Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-20T23:48:37.310475607Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:48:40.886886657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:48:40.897506207Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T23:48:42.310375558Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T23:48:47.310050798Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:48:49.23029298Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T23:48:49.348749015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:48:49.357274875Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-20T23:48:52.286581544Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:48:52.286618616Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:48:52.310130688Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-20T23:48:52.310146409Z caller=checkpoint.go:612 msg="starting checkpoint"
level=debug ts=2026-04-20T23:48:52.310161771Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:48:52.310170946Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:48:52.310184943Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:48:52.31031624Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=debug ts=2026-04-20T23:48:52.310523778Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
ts=2026-04-20T23:48:54.83245457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:53872"
level=debug ts=2026-04-20T23:48:56.113573489Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-20T23:48:57.309762742Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:48:58.769676154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:42844"
level=debug ts=2026-04-20T23:48:59.916666462Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-20T23:49:02.309517828Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:49:03.719926227Z caller=checkpoint.go:378 msg="writing series" size="253 kB"
ts=2026-04-20T23:49:06.959109402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-20T23:49:07.310653369Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:49:07.52242759Z caller=checkpoint.go:378 msg="writing series" size="42 kB"
level=debug ts=2026-04-20T23:49:11.325387429Z caller=checkpoint.go:378 msg="writing series" size="624 kB"
level=warn ts=2026-04-20T23:49:12.310083806Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:49:13.183032759Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T23:49:15.128216154Z caller=checkpoint.go:378 msg="writing series" size="348 kB"
level=debug ts=2026-04-20T23:49:15.128248678Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=7
level=debug ts=2026-04-20T23:49:16.589895666Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T23:49:17.309754558Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:49:18.931053358Z caller=checkpoint.go:378 msg="writing series" size="546 kB"
level=info ts=2026-04-20T23:49:22.310326615Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T23:49:22.310345608Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:49:22.310361079Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:49:22.310371132Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T23:49:22.733310339Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
level=debug ts=2026-04-20T23:49:26.536344669Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=warn ts=2026-04-20T23:49:27.309836892Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:49:28.771914827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:42720"
level=info ts=2026-04-20T23:49:29.394162433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:49:29.402112844Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:49:30.33948646Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=warn ts=2026-04-20T23:49:32.309578082Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:49:32.716615084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:49:32.724207495Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-20T23:49:34.141807452Z caller=checkpoint.go:378 msg="writing series" size="72 kB"
ts=2026-04-20T23:49:36.960893949Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-20T23:49:37.310402025Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:49:37.952850222Z caller=checkpoint.go:378 msg="writing series" size="12 MB"
level=debug ts=2026-04-20T23:49:37.952877745Z caller=checkpoint.go:390 msg="flushing series" totalSize="13 MB" series=6
level=debug ts=2026-04-20T23:49:41.352398139Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:49:41.747213894Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=warn ts=2026-04-20T23:49:42.310013122Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:49:45.551151428Z caller=checkpoint.go:378 msg="writing series" size="574 kB"
level=warn ts=2026-04-20T23:49:47.309684127Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:49:49.352948368Z caller=checkpoint.go:378 msg="writing series" size="51 kB"
level=info ts=2026-04-20T23:49:52.287305532Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:49:52.288391277Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:49:52.309769561Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:49:52.309793894Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:49:52.309802632Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:49:52.30982761Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:49:53.156445885Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=debug ts=2026-04-20T23:49:56.959339845Z caller=checkpoint.go:378 msg="writing series" size="76 kB"
level=warn ts=2026-04-20T23:49:57.309944088Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:49:58.774258217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:45670"
level=debug ts=2026-04-20T23:49:59.365938894Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T23:50:00.761725689Z caller=checkpoint.go:378 msg="writing series" size="240 kB"
level=warn ts=2026-04-20T23:50:02.310275598Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:50:04.564241379Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
ts=2026-04-20T23:50:06.96275337Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-20T23:50:07.310297632Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:50:08.367651848Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=debug ts=2026-04-20T23:50:12.170216579Z caller=checkpoint.go:378 msg="writing series" size="7.7 kB"
level=warn ts=2026-04-20T23:50:12.310352441Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:50:15.047723912Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T23:50:15.909986903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:50:15.92108641Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-20T23:50:15.973186193Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-20T23:50:17.310092645Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:50:19.776031232Z caller=checkpoint.go:378 msg="writing series" size="98 kB"
level=debug ts=2026-04-20T23:50:19.776055995Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-04-20T23:50:22.310391838Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:50:22.31054566Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:50:22.310560647Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:50:22.310570827Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T23:50:23.578557344Z caller=checkpoint.go:378 msg="writing series" size="176 kB"
ts=2026-04-20T23:50:24.837911184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:42734"
level=warn ts=2026-04-20T23:50:27.310100282Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:50:27.38154701Z caller=checkpoint.go:378 msg="writing series" size="300 kB"
level=info ts=2026-04-20T23:50:28.431676459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:50:28.443450657Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-20T23:50:28.776472846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:43246"
level=debug ts=2026-04-20T23:50:31.184100217Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=warn ts=2026-04-20T23:50:32.310234206Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:50:34.986857275Z caller=checkpoint.go:378 msg="writing series" size="4.1 kB"
level=debug ts=2026-04-20T23:50:35.427288208Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T23:50:35.429378964Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T23:50:36.965069688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-20T23:50:37.309517912Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:50:38.790242552Z caller=checkpoint.go:378 msg="writing series" size="102 kB"
level=warn ts=2026-04-20T23:50:42.310134633Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:50:42.592594071Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=debug ts=2026-04-20T23:50:46.395415065Z caller=checkpoint.go:378 msg="writing series" size="81 kB"
level=warn ts=2026-04-20T23:50:47.3100963Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:50:50.198796596Z caller=checkpoint.go:378 msg="writing series" size="81 kB"
level=debug ts=2026-04-20T23:50:50.892049785Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T23:50:52.286589435Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:50:52.287697794Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:50:52.310162813Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:50:52.310204747Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:50:52.310213754Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:50:52.310203816Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:50:54.001328021Z caller=checkpoint.go:378 msg="writing series" size="6.7 kB"
level=warn ts=2026-04-20T23:50:57.310104696Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:50:57.803974052Z caller=checkpoint.go:378 msg="writing series" size="392 kB"
level=debug ts=2026-04-20T23:50:57.80400141Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=10
ts=2026-04-20T23:50:58.778540344Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:55164"
level=debug ts=2026-04-20T23:51:01.613405997Z caller=checkpoint.go:378 msg="writing series" size="6.5 MB"
level=debug ts=2026-04-20T23:51:01.613684504Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.5 MB" series=1
level=warn ts=2026-04-20T23:51:02.309884805Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:51:03.78867629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:51:03.800529572Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-20T23:51:05.409194256Z caller=checkpoint.go:378 msg="writing series" size="514 B"
ts=2026-04-20T23:51:06.96734172Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-20T23:51:07.310104602Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:51:09.213037398Z caller=checkpoint.go:378 msg="writing series" size="511 kB"
level=info ts=2026-04-20T23:51:09.721482593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:51:09.728483654Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:51:09.728514389Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T23:51:09.728546176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:51:09.735259528Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T23:51:12.310306474Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:51:13.015309802Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=debug ts=2026-04-20T23:51:16.81815036Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=debug ts=2026-04-20T23:51:16.97672888Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T23:51:17.310463934Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:51:20.621181357Z caller=checkpoint.go:378 msg="writing series" size="99 kB"
level=info ts=2026-04-20T23:51:22.31028317Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:51:22.310318386Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:51:22.310328863Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:51:22.31032473Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:51:24.045835956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:51:24.055618474Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:51:24.42385445Z caller=checkpoint.go:378 msg="writing series" size="142 kB"
level=warn ts=2026-04-20T23:51:27.309529181Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:51:28.226711885Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=debug ts=2026-04-20T23:51:32.029400359Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-20T23:51:32.310195726Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:51:34.658713959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T23:51:35.832291522Z caller=checkpoint.go:378 msg="writing series" size="809 B"
ts=2026-04-20T23:51:36.969162099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-20T23:51:37.310333535Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:51:39.634879188Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=warn ts=2026-04-20T23:51:42.309719602Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:51:42.400055133Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:51:43.437504405Z caller=checkpoint.go:378 msg="writing series" size="322 kB"
level=debug ts=2026-04-20T23:51:43.43752909Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=debug ts=2026-04-20T23:51:47.240512899Z caller=checkpoint.go:378 msg="writing series" size="155 kB"
level=warn ts=2026-04-20T23:51:47.310262492Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:51:51.044021897Z caller=checkpoint.go:378 msg="writing series" size="317 kB"
level=info ts=2026-04-20T23:51:52.286526123Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:51:52.28768684Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:51:52.310191536Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:51:52.310214007Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:51:52.310222349Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:51:52.310242983Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:51:54.844245069Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:59360"
level=debug ts=2026-04-20T23:51:54.847031324Z caller=checkpoint.go:378 msg="writing series" size="365 kB"
level=warn ts=2026-04-20T23:51:57.309992909Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:51:58.161883808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:51:58.171703227Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-20T23:51:58.649090449Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=warn ts=2026-04-20T23:52:02.310378591Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:52:02.451810041Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=info ts=2026-04-20T23:52:02.503917025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:52:02.515197233Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:52:06.254925948Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
ts=2026-04-20T23:52:06.971864831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-20T23:52:07.309791842Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:52:10.057877989Z caller=checkpoint.go:378 msg="writing series" size="208 kB"
level=debug ts=2026-04-20T23:52:10.057910964Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-04-20T23:52:12.309968036Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:52:13.847393318Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:52:13.860581762Z caller=checkpoint.go:378 msg="writing series" size="128 kB"
level=warn ts=2026-04-20T23:52:17.310266234Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:52:17.662956589Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=debug ts=2026-04-20T23:52:18.339158819Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T23:52:21.465481018Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=info ts=2026-04-20T23:52:22.309661086Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:52:22.309699033Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:52:22.30971118Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:52:22.309709536Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:52:25.269053023Z caller=checkpoint.go:378 msg="writing series" size="84 kB"
level=warn ts=2026-04-20T23:52:27.309510525Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:52:28.783708259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:56920"
level=debug ts=2026-04-20T23:52:29.07169838Z caller=checkpoint.go:378 msg="writing series" size="123 kB"
level=warn ts=2026-04-20T23:52:32.309496276Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:52:32.874786268Z caller=checkpoint.go:378 msg="writing series" size="8.2 kB"
level=debug ts=2026-04-20T23:52:36.677241396Z caller=checkpoint.go:378 msg="writing series" size="90 kB"
ts=2026-04-20T23:52:36.973299171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-20T23:52:37.30983777Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:52:40.481331761Z caller=checkpoint.go:378 msg="writing series" size="1.9 MB"
level=debug ts=2026-04-20T23:52:40.481356998Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.4 MB" series=8
level=info ts=2026-04-20T23:52:41.413904404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:52:41.424214079Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-20T23:52:42.309717656Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:52:43.220935189Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T23:52:44.043373043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:52:44.050770453Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:52:44.282770282Z caller=checkpoint.go:378 msg="writing series" size="195 kB"
level=warn ts=2026-04-20T23:52:47.310362576Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:52:48.085572205Z caller=checkpoint.go:378 msg="writing series" size="238 kB"
level=debug ts=2026-04-20T23:52:51.888246045Z caller=checkpoint.go:378 msg="writing series" size="2.5 kB"
level=info ts=2026-04-20T23:52:52.286384038Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:52:52.286649769Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:52:52.310217093Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:52:52.310246491Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:52:52.310266053Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:52:52.310260056Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:52:55.691799207Z caller=checkpoint.go:378 msg="writing series" size="336 kB"
level=warn ts=2026-04-20T23:52:57.310114621Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:52:59.494899655Z caller=checkpoint.go:378 msg="writing series" size="446 kB"
level=debug ts=2026-04-20T23:52:59.49492612Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-04-20T23:53:02.310041961Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:53:03.296752319Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=debug ts=2026-04-20T23:53:03.426196601Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T23:53:06.975328169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=debug ts=2026-04-20T23:53:07.099942313Z caller=checkpoint.go:378 msg="writing series" size="478 kB"
level=warn ts=2026-04-20T23:53:07.309938417Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:53:10.902830752Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=debug ts=2026-04-20T23:53:11.512675533Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T23:53:12.309821194Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:53:14.705006572Z caller=checkpoint.go:378 msg="writing series" size="186 kB"
level=warn ts=2026-04-20T23:53:17.309921218Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:53:18.508490531Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=info ts=2026-04-20T23:53:22.309669608Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:53:22.309698699Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:53:22.309706582Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:53:22.309709246Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:53:22.310773108Z caller=checkpoint.go:390 msg="flushing series" totalSize="706 kB" series=5
level=info ts=2026-04-20T23:53:22.313839332Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-04-20T23:53:22.314775699Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.004317785s
level=warn ts=2026-04-20T23:53:27.310446429Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:53:28.382278498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:53:28.393805847Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:53:28.393831456Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T23:53:28.393863371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:53:28.400339936Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-20T23:53:28.787496773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:37670"
level=warn ts=2026-04-20T23:53:32.310493966Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:53:36.977292243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-20T23:53:37.31064643Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:53:41.819583018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:53:41.829875006Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:53:41.907311423Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T23:53:42.310446814Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:53:46.188876327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:53:46.196313837Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-20T23:53:47.310286358Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:53:52.286446602Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:53:52.286876656Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:53:52.309348136Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-20T23:53:52.309484773Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=info ts=2026-04-20T23:53:52.309495982Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:53:52.309514456Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:53:52.309523121Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:53:52.309538552Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:53:52.309696907Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
level=debug ts=2026-04-20T23:53:54.424578518Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T23:53:54.851813058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:47042"
level=debug ts=2026-04-20T23:53:56.113062182Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-20T23:53:57.310229467Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:53:59.915539261Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-20T23:54:02.30950811Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:54:03.718761001Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=debug ts=2026-04-20T23:54:06.45744799Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-20T23:54:06.978693886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-20T23:54:07.309944762Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:54:07.521180552Z caller=checkpoint.go:378 msg="writing series" size="44 kB"
level=debug ts=2026-04-20T23:54:11.324323712Z caller=checkpoint.go:378 msg="writing series" size="624 kB"
level=warn ts=2026-04-20T23:54:12.310036843Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:54:15.127399733Z caller=checkpoint.go:378 msg="writing series" size="655 kB"
level=debug ts=2026-04-20T23:54:15.127434993Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=7
level=warn ts=2026-04-20T23:54:17.309601885Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:54:18.930111324Z caller=checkpoint.go:378 msg="writing series" size="814 kB"
level=info ts=2026-04-20T23:54:20.769273113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:54:20.778430276Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-20T23:54:22.31037496Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:54:22.310405677Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-20T23:54:22.310392257Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:54:22.31041421Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T23:54:22.733287755Z caller=checkpoint.go:378 msg="writing series" size="120 kB"
level=debug ts=2026-04-20T23:54:26.535733013Z caller=checkpoint.go:378 msg="writing series" size="55 kB"
level=warn ts=2026-04-20T23:54:27.310132647Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:54:27.753143787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:54:27.760424928Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:54:30.337914764Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=debug ts=2026-04-20T23:54:30.337956442Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-04-20T23:54:32.309849329Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:54:34.14156731Z caller=checkpoint.go:378 msg="writing series" size="80 kB"
level=debug ts=2026-04-20T23:54:34.648450229Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T23:54:34.650361347Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-20T23:54:35.737573Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-20T23:54:36.980995268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-20T23:54:37.31044262Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:54:37.961841069Z caller=checkpoint.go:378 msg="writing series" size="17 MB"
level=debug ts=2026-04-20T23:54:37.961871923Z caller=checkpoint.go:390 msg="flushing series" totalSize="17 MB" series=2
level=debug ts=2026-04-20T23:54:41.74729845Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=warn ts=2026-04-20T23:54:42.309742407Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:54:45.549948176Z caller=checkpoint.go:378 msg="writing series" size="683 kB"
level=warn ts=2026-04-20T23:54:47.310455722Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:54:49.352782337Z caller=checkpoint.go:378 msg="writing series" size="84 kB"
level=info ts=2026-04-20T23:54:52.28675536Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:54:52.287772529Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:54:52.310224783Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:54:52.31025139Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:54:52.310259705Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:54:52.310278039Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:54:53.155446554Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=debug ts=2026-04-20T23:54:56.957640381Z caller=checkpoint.go:378 msg="writing series" size="76 kB"
level=warn ts=2026-04-20T23:54:57.310316143Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:55:00.486785863Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:55:00.760823516Z caller=checkpoint.go:378 msg="writing series" size="89 kB"
level=warn ts=2026-04-20T23:55:02.310081099Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:55:04.564098394Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
ts=2026-04-20T23:55:06.98345897Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-20T23:55:07.309807715Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:55:08.009909519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:55:08.020876526Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-20T23:55:08.366294103Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=debug ts=2026-04-20T23:55:11.045047577Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T23:55:12.169708967Z caller=checkpoint.go:378 msg="writing series" size="8.6 kB"
level=warn ts=2026-04-20T23:55:12.310166051Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:55:15.972139386Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-20T23:55:17.30962137Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:55:19.774834231Z caller=checkpoint.go:378 msg="writing series" size="116 kB"
level=debug ts=2026-04-20T23:55:19.774860614Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=11
level=info ts=2026-04-20T23:55:19.82708395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:55:19.839237192Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T23:55:22.309965092Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:55:22.309967358Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:55:22.309993561Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:55:22.310004109Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T23:55:23.578139671Z caller=checkpoint.go:378 msg="writing series" size="203 kB"
level=warn ts=2026-04-20T23:55:27.310572622Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:55:27.38101874Z caller=checkpoint.go:378 msg="writing series" size="73 kB"
ts=2026-04-20T23:55:28.717306698Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:44644"
level=debug ts=2026-04-20T23:55:31.183405785Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=warn ts=2026-04-20T23:55:32.309705274Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:55:34.986164624Z caller=checkpoint.go:378 msg="writing series" size="4.1 kB"
level=debug ts=2026-04-20T23:55:36.123511079Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-20T23:55:36.985760878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-20T23:55:37.310146714Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:55:38.788976066Z caller=checkpoint.go:378 msg="writing series" size="157 kB"
level=warn ts=2026-04-20T23:55:42.310151243Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:55:42.591701262Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=debug ts=2026-04-20T23:55:46.395379191Z caller=checkpoint.go:378 msg="writing series" size="146 kB"
level=warn ts=2026-04-20T23:55:47.309572023Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:55:50.197315482Z caller=checkpoint.go:378 msg="writing series" size="81 kB"
level=info ts=2026-04-20T23:55:52.2872054Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:55:52.288344027Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:55:52.309649012Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:55:52.309678472Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:55:52.309687999Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:55:52.309693488Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:55:54.000196541Z caller=checkpoint.go:378 msg="writing series" size="6.7 kB"
level=warn ts=2026-04-20T23:55:57.310579302Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:55:57.802836521Z caller=checkpoint.go:378 msg="writing series" size="276 kB"
level=debug ts=2026-04-20T23:55:57.802865548Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.0 MB" series=10
ts=2026-04-20T23:55:58.796423417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:37344"
level=debug ts=2026-04-20T23:56:01.612791759Z caller=checkpoint.go:378 msg="writing series" size="7.1 MB"
level=debug ts=2026-04-20T23:56:01.612818681Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.1 MB" series=1
level=warn ts=2026-04-20T23:56:02.310343588Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:56:03.713275954Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:56:05.409287183Z caller=checkpoint.go:378 msg="writing series" size="514 B"
level=info ts=2026-04-20T23:56:06.816564064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.7 MB" total_uncomp="135 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:56:06.824508211Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-20T23:56:06.987691757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-20T23:56:07.30955285Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:56:09.063333467Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T23:56:09.212143115Z caller=checkpoint.go:378 msg="writing series" size="524 kB"
level=warn ts=2026-04-20T23:56:12.31028132Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:56:13.01432677Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=debug ts=2026-04-20T23:56:16.816977424Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-20T23:56:17.309764166Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:56:17.442120792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:56:17.451739894Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:56:20.620072357Z caller=checkpoint.go:378 msg="writing series" size="105 kB"
level=info ts=2026-04-20T23:56:22.309766528Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:56:22.309797861Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:56:22.309806011Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:56:22.30980226Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:56:24.423400044Z caller=checkpoint.go:378 msg="writing series" size="142 kB"
level=warn ts=2026-04-20T23:56:27.309990697Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:56:28.226208024Z caller=checkpoint.go:378 msg="writing series" size="33 kB"
ts=2026-04-20T23:56:28.798143815Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:40260"
level=debug ts=2026-04-20T23:56:32.028867684Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-20T23:56:32.309808178Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:56:34.661928775Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:56:35.831697433Z caller=checkpoint.go:378 msg="writing series" size="809 B"
ts=2026-04-20T23:56:36.988843994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-20T23:56:37.310006912Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:56:39.634398963Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=warn ts=2026-04-20T23:56:42.309589946Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:56:43.437695904Z caller=checkpoint.go:378 msg="writing series" size="841 kB"
level=debug ts=2026-04-20T23:56:43.437722981Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=11
level=debug ts=2026-04-20T23:56:47.240259346Z caller=checkpoint.go:378 msg="writing series" size="180 kB"
level=warn ts=2026-04-20T23:56:47.310375715Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:56:48.405887366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="19 MB" avg_comp="2.7 MB" total_uncomp="159 MB" avg_uncomp="23 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:56:48.417881191Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-20T23:56:51.043358702Z caller=checkpoint.go:378 msg="writing series" size="449 kB"
level=info ts=2026-04-20T23:56:52.286650604Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:56:52.287143646Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:56:52.30959943Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:56:52.309620297Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:56:52.309649482Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:56:52.309656518Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:56:54.845293891Z caller=checkpoint.go:378 msg="writing series" size="365 kB"
level=warn ts=2026-04-20T23:56:57.310430463Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:56:58.648586167Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=debug ts=2026-04-20T23:57:01.258408913Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T23:57:02.310394411Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:57:02.450704986Z caller=checkpoint.go:378 msg="writing series" size="43 kB"
level=debug ts=2026-04-20T23:57:02.450732046Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5
level=debug ts=2026-04-20T23:57:04.31041988Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T23:57:06.25436077Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
ts=2026-04-20T23:57:06.989985208Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-20T23:57:07.309516674Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:57:10.057312109Z caller=checkpoint.go:378 msg="writing series" size="343 kB"
level=warn ts=2026-04-20T23:57:12.309771006Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:57:12.327885421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:57:12.336004992Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:57:13.860077427Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=warn ts=2026-04-20T23:57:17.309652266Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:57:17.66239038Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=debug ts=2026-04-20T23:57:21.464779419Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=info ts=2026-04-20T23:57:22.310327384Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:57:22.310361094Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:57:22.310369623Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:57:22.310361263Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:57:25.268889053Z caller=checkpoint.go:378 msg="writing series" size="125 kB"
level=warn ts=2026-04-20T23:57:27.310304384Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T23:57:28.725234678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:59848"
level=debug ts=2026-04-20T23:57:29.070688459Z caller=checkpoint.go:378 msg="writing series" size="169 kB"
level=warn ts=2026-04-20T23:57:32.310024057Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:57:32.873294334Z caller=checkpoint.go:378 msg="writing series" size="8.2 kB"
level=debug ts=2026-04-20T23:57:33.061981196Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:57:36.676264768Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=info ts=2026-04-20T23:57:36.930061193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="19 MB" avg_comp="2.7 MB" total_uncomp="159 MB" avg_uncomp="23 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:57:36.939921337Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-20T23:57:36.991803339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-20T23:57:37.309675593Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:57:40.480062857Z caller=checkpoint.go:378 msg="writing series" size="2.2 MB"
level=debug ts=2026-04-20T23:57:40.480088552Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.1 MB" series=10
level=warn ts=2026-04-20T23:57:42.309780104Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:57:43.363991434Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T23:57:44.281735063Z caller=checkpoint.go:378 msg="writing series" size="304 kB"
level=warn ts=2026-04-20T23:57:47.310103012Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:57:48.084899132Z caller=checkpoint.go:378 msg="writing series" size="391 kB"
level=debug ts=2026-04-20T23:57:51.892031755Z caller=checkpoint.go:378 msg="writing series" size="2.8 kB"
level=info ts=2026-04-20T23:57:52.286399411Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:57:52.286653655Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=warn ts=2026-04-20T23:57:52.310220629Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:57:52.310229546Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:57:52.310253764Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:57:52.310264621Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-20T23:57:54.83602936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:57:54.84478166Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:57:54.844813351Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T23:57:54.844842572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:57:54.851247084Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:57:55.690444041Z caller=checkpoint.go:378 msg="writing series" size="336 kB"
level=warn ts=2026-04-20T23:57:57.309567462Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:57:59.493507674Z caller=checkpoint.go:378 msg="writing series" size="502 kB"
level=debug ts=2026-04-20T23:57:59.493533253Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=5
level=warn ts=2026-04-20T23:58:02.310553593Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:58:02.740935996Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:58:03.29598963Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
ts=2026-04-20T23:58:06.993827099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=debug ts=2026-04-20T23:58:07.099305663Z caller=checkpoint.go:378 msg="writing series" size="566 kB"
level=warn ts=2026-04-20T23:58:07.309895464Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:58:10.902078259Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=warn ts=2026-04-20T23:58:12.309775381Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:58:12.418138511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:58:12.430819603Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:58:14.704540574Z caller=checkpoint.go:378 msg="writing series" size="187 kB"
level=warn ts=2026-04-20T23:58:17.309891603Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:58:18.507512861Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=info ts=2026-04-20T23:58:18.620035265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="19 MB" avg_comp="2.7 MB" total_uncomp="159 MB" avg_uncomp="23 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:58:18.627612302Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-20T23:58:22.309829001Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:58:22.309826078Z caller=checkpoint.go:390 msg="flushing series" totalSize="795 kB" series=5
level=info ts=2026-04-20T23:58:22.309834391Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:58:22.309858628Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:58:22.309867819Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-20T23:58:22.318007774Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-04-20T23:58:22.319030569Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.009407895s
ts=2026-04-20T23:58:24.867637243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:53480"
level=warn ts=2026-04-20T23:58:27.309888902Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:58:30.360117988Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T23:58:32.309517709Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:58:35.674592809Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T23:58:35.676190769Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T23:58:36.994902672Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-20T23:58:37.310267952Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:58:37.723598913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:58:37.732410049Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T23:58:42.30984045Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T23:58:47.310137142Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:58:52.286910928Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:58:52.287953002Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:58:52.309333076Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-20T23:58:52.309491415Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=info ts=2026-04-20T23:58:52.309499885Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:58:52.309517949Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:58:52.309526396Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:58:52.309541549Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:58:52.309690153Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
ts=2026-04-20T23:58:54.868686225Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:37652"
level=debug ts=2026-04-20T23:58:56.112881065Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-20T23:58:57.310211225Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:58:59.915871544Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-04-20T23:59:01.65160177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="19 MB" avg_comp="2.7 MB" total_uncomp="159 MB" avg_uncomp="23 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:59:01.65946403Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-20T23:59:02.310486166Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:59:02.514798859Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T23:59:03.718893005Z caller=checkpoint.go:378 msg="writing series" size="140 kB"
ts=2026-04-20T23:59:06.997105473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-20T23:59:07.310457041Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:59:07.521545555Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=debug ts=2026-04-20T23:59:11.325051993Z caller=checkpoint.go:378 msg="writing series" size="624 kB"
level=warn ts=2026-04-20T23:59:12.309714503Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:59:15.127884526Z caller=checkpoint.go:378 msg="writing series" size="826 kB"
level=debug ts=2026-04-20T23:59:15.127909215Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=7
level=warn ts=2026-04-20T23:59:17.309741776Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:59:18.40820464Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T23:59:18.930360253Z caller=checkpoint.go:378 msg="writing series" size="951 kB"
level=info ts=2026-04-20T23:59:22.310056587Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:59:22.310090812Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:59:22.310099Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:59:22.310086966Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:59:22.732716472Z caller=checkpoint.go:378 msg="writing series" size="146 kB"
level=debug ts=2026-04-20T23:59:22.732753917Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=2
level=debug ts=2026-04-20T23:59:26.535615999Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=warn ts=2026-04-20T23:59:27.310405725Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:59:30.338689006Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=warn ts=2026-04-20T23:59:32.310103333Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T23:59:33.21290621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T23:59:33.223195873Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T23:59:34.141102103Z caller=checkpoint.go:378 msg="writing series" size="89 kB"
level=debug ts=2026-04-20T23:59:35.693530927Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-20T23:59:36.99890537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-20T23:59:37.309721953Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:59:37.960582733Z caller=checkpoint.go:378 msg="writing series" size="21 MB"
level=debug ts=2026-04-20T23:59:37.960618961Z caller=checkpoint.go:390 msg="flushing series" totalSize="22 MB" series=4
level=debug ts=2026-04-20T23:59:41.746965708Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=warn ts=2026-04-20T23:59:42.309497346Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:59:45.550390549Z caller=checkpoint.go:378 msg="writing series" size="775 kB"
level=warn ts=2026-04-20T23:59:47.309959063Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:59:49.353040949Z caller=checkpoint.go:378 msg="writing series" size="114 kB"
level=info ts=2026-04-20T23:59:52.170503008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="19 MB" avg_comp="2.7 MB" total_uncomp="159 MB" avg_uncomp="23 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:59:52.180070525Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:59:52.180098849Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T23:59:52.180131758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="19 MB" avg_comp="2.7 MB" total_uncomp="159 MB" avg_uncomp="23 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T23:59:52.186684103Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-20T23:59:52.28650362Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T23:59:52.286899812Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-20T23:59:52.309445419Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T23:59:52.309487258Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T23:59:52.309495453Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T23:59:52.309496215Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T23:59:53.155658281Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
ts=2026-04-20T23:59:54.87250687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:38638"
level=debug ts=2026-04-20T23:59:56.958282946Z caller=checkpoint.go:378 msg="writing series" size="77 kB"
level=warn ts=2026-04-20T23:59:57.309535294Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:00:00.760921689Z caller=checkpoint.go:378 msg="writing series" size="169 kB"
level=debug ts=2026-04-21T00:00:00.760957046Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=6
level=debug ts=2026-04-21T00:00:01.380156996Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T00:00:02.310195178Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:00:02.625099752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="19 MB" avg_comp="2.7 MB" total_uncomp="159 MB" avg_uncomp="23 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:00:02.636472318Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:00:04.563726645Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
ts=2026-04-21T00:00:07.00103862Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:00:07.309849402Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:00:08.36630528Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=debug ts=2026-04-21T00:00:08.97314969Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:00:12.16920323Z caller=checkpoint.go:378 msg="writing series" size="9.8 kB"
level=warn ts=2026-04-21T00:00:12.310236123Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:00:15.972055488Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-21T00:00:17.310005892Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:00:19.775228976Z caller=checkpoint.go:378 msg="writing series" size="129 kB"
level=info ts=2026-04-21T00:00:19.966851526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:00:19.976748496Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T00:00:22.309963154Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:00:22.30996312Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:00:22.31001295Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:00:22.310030184Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:00:23.577774847Z caller=checkpoint.go:378 msg="writing series" size="190 kB"
level=info ts=2026-04-21T00:00:23.942673699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="19 MB" avg_comp="2.7 MB" total_uncomp="159 MB" avg_uncomp="23 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:00:23.95000482Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-21T00:00:27.310057194Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:00:27.380398821Z caller=checkpoint.go:378 msg="writing series" size="139 kB"
ts=2026-04-21T00:00:28.735028938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:52646"
level=debug ts=2026-04-21T00:00:31.183715633Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=warn ts=2026-04-21T00:00:32.310277449Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:00:34.986756382Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
ts=2026-04-21T00:00:37.00328196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:00:37.310130171Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:00:38.789519126Z caller=checkpoint.go:378 msg="writing series" size="206 kB"
level=warn ts=2026-04-21T00:00:42.310486307Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:00:42.592475598Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=debug ts=2026-04-21T00:00:42.698349526Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:00:46.394491872Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=debug ts=2026-04-21T00:00:46.394522339Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=12
level=warn ts=2026-04-21T00:00:47.310416221Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:00:50.197669294Z caller=checkpoint.go:378 msg="writing series" size="81 kB"
level=info ts=2026-04-21T00:00:52.286381016Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:00:52.286795645Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=debug ts=2026-04-21T00:00:52.288121819Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-21T00:00:52.288174635Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974143 n=1
level=debug ts=2026-04-21T00:00:52.288236542Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974143 n=1
level=info ts=2026-04-21T00:00:52.309500114Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:00:52.309511483Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:00:52.309535619Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:00:52.309547159Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:00:53.454304354Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T00:00:54.000619109Z caller=checkpoint.go:378 msg="writing series" size="6.7 kB"
level=warn ts=2026-04-21T00:00:57.309779977Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:00:57.803507339Z caller=checkpoint.go:378 msg="writing series" size="415 kB"
ts=2026-04-21T00:00:58.737571509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:34746"
level=debug ts=2026-04-21T00:01:01.610580229Z caller=checkpoint.go:378 msg="writing series" size="7.6 MB"
level=debug ts=2026-04-21T00:01:01.610609468Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.1 MB" series=4
level=warn ts=2026-04-21T00:01:02.309767619Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:01:05.408943062Z caller=checkpoint.go:378 msg="writing series" size="514 B"
ts=2026-04-21T00:01:07.004933582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:01:07.310429572Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:01:09.212130264Z caller=checkpoint.go:378 msg="writing series" size="512 kB"
level=info ts=2026-04-21T00:01:10.349981835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:01:10.359755952Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:01:10.538421911Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:01:12.310345669Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:01:13.014386515Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=debug ts=2026-04-21T00:01:16.817129329Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-21T00:01:17.309764702Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:01:19.152779475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="19 MB" avg_comp="2.7 MB" total_uncomp="159 MB" avg_uncomp="23 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:01:19.162358307Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:01:20.61979026Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=info ts=2026-04-21T00:01:22.310081302Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:01:22.31010357Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:01:22.310136102Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:01:22.310154576Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:01:24.423325726Z caller=checkpoint.go:378 msg="writing series" size="142 kB"
level=warn ts=2026-04-21T00:01:27.309805314Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:01:28.161576403Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T00:01:28.225870098Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=debug ts=2026-04-21T00:01:32.028713989Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-21T00:01:32.309821855Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:01:35.83115033Z caller=checkpoint.go:378 msg="writing series" size="809 B"
ts=2026-04-21T00:01:37.007053657Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:01:37.309619898Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:01:39.634562543Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=warn ts=2026-04-21T00:01:42.31038622Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:01:43.437412132Z caller=checkpoint.go:378 msg="writing series" size="841 kB"
level=debug ts=2026-04-21T00:01:43.437440572Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=11
level=debug ts=2026-04-21T00:01:45.008332861Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:01:47.240401488Z caller=checkpoint.go:378 msg="writing series" size="205 kB"
level=warn ts=2026-04-21T00:01:47.310410735Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:01:51.042796445Z caller=checkpoint.go:378 msg="writing series" size="337 kB"
level=info ts=2026-04-21T00:01:52.286759329Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:01:52.288060606Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:01:52.309443462Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:01:52.309514611Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:01:52.309550742Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:01:52.309566642Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:01:54.476994034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:01:54.485278549Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:01:54.845197734Z caller=checkpoint.go:378 msg="writing series" size="333 kB"
ts=2026-04-21T00:01:54.879247509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:55622"
level=warn ts=2026-04-21T00:01:57.309778901Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:01:58.648082654Z caller=checkpoint.go:378 msg="writing series" size="55 kB"
level=warn ts=2026-04-21T00:02:02.309761644Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:02:02.451100342Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=debug ts=2026-04-21T00:02:06.254373093Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
ts=2026-04-21T00:02:07.009299313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:02:07.310486225Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:02:10.056460557Z caller=checkpoint.go:378 msg="writing series" size="321 kB"
level=debug ts=2026-04-21T00:02:10.056485868Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=7
level=warn ts=2026-04-21T00:02:12.30979975Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:02:13.859521939Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=debug ts=2026-04-21T00:02:15.636420379Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T00:02:15.638319265Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T00:02:17.074373015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:02:17.082171261Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-21T00:02:17.310419283Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:02:17.66205176Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=debug ts=2026-04-21T00:02:19.201597646Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:02:21.464942935Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=info ts=2026-04-21T00:02:22.309957292Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:02:22.309966287Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:02:22.30998802Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:02:22.309996062Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:02:25.268439232Z caller=checkpoint.go:378 msg="writing series" size="193 kB"
level=warn ts=2026-04-21T00:02:27.309740118Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:02:28.743049826Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:35680"
level=debug ts=2026-04-21T00:02:29.071008386Z caller=checkpoint.go:378 msg="writing series" size="210 kB"
level=warn ts=2026-04-21T00:02:32.310350784Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:02:32.873823583Z caller=checkpoint.go:378 msg="writing series" size="8.2 kB"
level=debug ts=2026-04-21T00:02:36.676319372Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
ts=2026-04-21T00:02:37.011851464Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:02:37.310059521Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:02:40.48136442Z caller=checkpoint.go:378 msg="writing series" size="2.2 MB"
level=debug ts=2026-04-21T00:02:40.481387388Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.0 MB" series=8
level=warn ts=2026-04-21T00:02:42.30953737Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:02:43.681213569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:02:43.692912648Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:02:44.282462449Z caller=checkpoint.go:378 msg="writing series" size="429 kB"
level=warn ts=2026-04-21T00:02:47.309783488Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:02:48.085537733Z caller=checkpoint.go:378 msg="writing series" size="869 kB"
level=debug ts=2026-04-21T00:02:48.085564384Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=2
level=debug ts=2026-04-21T00:02:49.183388662Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:02:51.887791567Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=info ts=2026-04-21T00:02:52.286372243Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:02:52.28689708Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:02:52.310148527Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:02:52.310176936Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:02:52.310185208Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:02:52.310204853Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:02:54.882851428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:33020"
level=debug ts=2026-04-21T00:02:55.690428569Z caller=checkpoint.go:378 msg="writing series" size="336 kB"
level=debug ts=2026-04-21T00:02:55.873259092Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T00:02:57.31019557Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:02:57.799432018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:02:57.809308983Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-21T00:02:58.745333865Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:40314"
ts=2026-04-21T00:02:58.820911411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:43546"
level=debug ts=2026-04-21T00:02:59.494749312Z caller=checkpoint.go:378 msg="writing series" size="585 kB"
level=warn ts=2026-04-21T00:03:02.310122643Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:03:03.296099144Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
ts=2026-04-21T00:03:07.014386313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=debug ts=2026-04-21T00:03:07.098979243Z caller=checkpoint.go:378 msg="writing series" size="560 kB"
level=debug ts=2026-04-21T00:03:07.099005494Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=5
level=warn ts=2026-04-21T00:03:07.31007182Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:03:10.901826447Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=warn ts=2026-04-21T00:03:12.309797733Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:03:14.704292117Z caller=checkpoint.go:378 msg="writing series" size="188 kB"
level=warn ts=2026-04-21T00:03:17.310468595Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:03:18.507533582Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=debug ts=2026-04-21T00:03:20.244451126Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:03:20.244519249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="377 kB" avg_comp="377 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:03:20.251417759Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:03:22.309994009Z caller=checkpoint.go:390 msg="flushing series" totalSize="226 kB" series=3
level=info ts=2026-04-21T00:03:22.310028689Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:03:22.31005316Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:03:22.310061376Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:03:22.310053623Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:03:22.315493292Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-04-21T00:03:22.316522021Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.006898828s
ts=2026-04-21T00:03:24.884163843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:37364"
level=warn ts=2026-04-21T00:03:27.309539243Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:03:28.747324632Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:41276"
level=info ts=2026-04-21T00:03:31.799679413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:03:31.808772306Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T00:03:32.301360604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:03:32.305569282Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-21T00:03:32.309675216Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:03:37.02036533Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:03:37.30975352Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T00:03:42.309486324Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:03:45.332832006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:03:45.343577289Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-21T00:03:47.313732212Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:03:49.974505373Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:03:52.127070135Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T00:03:52.28623591Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:03:52.287295605Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T00:03:52.309735042Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-04-21T00:03:52.309774069Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:03:52.309790186Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:03:52.309810058Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:03:52.309818737Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T00:03:52.309919448Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=debug ts=2026-04-21T00:03:52.310130322Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
level=debug ts=2026-04-21T00:03:56.113658253Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=info ts=2026-04-21T00:03:56.696464817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:03:56.701905073Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-21T00:03:57.310231485Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:03:59.91664207Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-21T00:04:02.310268436Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:04:03.719550351Z caller=checkpoint.go:378 msg="writing series" size="191 kB"
ts=2026-04-21T00:04:07.022602428Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:04:07.310174987Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:04:07.521467187Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=debug ts=2026-04-21T00:04:11.325076322Z caller=checkpoint.go:378 msg="writing series" size="709 kB"
level=warn ts=2026-04-21T00:04:12.310375533Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:04:15.127995482Z caller=checkpoint.go:378 msg="writing series" size="1.0 MB"
level=debug ts=2026-04-21T00:04:15.128026287Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.0 MB" series=7
level=debug ts=2026-04-21T00:04:15.471471075Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:04:17.310019078Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:04:18.93093575Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-21T00:04:18.93096202Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-04-21T00:04:22.309579882Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:04:22.30961577Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:04:22.309643558Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:04:22.309677202Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:04:22.733593916Z caller=checkpoint.go:378 msg="writing series" size="188 kB"
level=debug ts=2026-04-21T00:04:26.536295975Z caller=checkpoint.go:378 msg="writing series" size="66 kB"
level=warn ts=2026-04-21T00:04:27.310146201Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:04:28.825488438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:43544"
level=debug ts=2026-04-21T00:04:30.338720275Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=info ts=2026-04-21T00:04:31.40878102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:04:31.417582718Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:04:31.417607607Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:04:31.417651904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:04:31.424007448Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T00:04:32.310488032Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:04:33.726054018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:04:33.733198799Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:04:34.141551028Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
ts=2026-04-21T00:04:37.024969001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:04:37.309658554Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:04:37.974642922Z caller=checkpoint.go:378 msg="writing series" size="23 MB"
level=debug ts=2026-04-21T00:04:37.974675693Z caller=checkpoint.go:390 msg="flushing series" totalSize="23 MB" series=5
level=debug ts=2026-04-21T00:04:41.746835256Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=warn ts=2026-04-21T00:04:42.310266734Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:04:43.336142984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:04:43.340344897Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:04:44.024522272Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T00:04:45.550868551Z caller=checkpoint.go:378 msg="writing series" size="999 kB"
level=debug ts=2026-04-21T00:04:45.550893935Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.0 MB" series=2
level=debug ts=2026-04-21T00:04:45.782408614Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:04:46.148027512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:04:46.15611792Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T00:04:47.310193614Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:04:49.352812321Z caller=checkpoint.go:378 msg="writing series" size="146 kB"
level=info ts=2026-04-21T00:04:52.286547261Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:04:52.286941101Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:04:52.309440935Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:04:52.309469398Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:04:52.309478682Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:04:52.30951154Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:04:53.155608014Z caller=checkpoint.go:378 msg="writing series" size="7.2 kB"
ts=2026-04-21T00:04:54.889170609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:57630"
level=debug ts=2026-04-21T00:04:56.95810371Z caller=checkpoint.go:378 msg="writing series" size="77 kB"
level=warn ts=2026-04-21T00:04:57.310217304Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:04:58.827267211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:56496"
level=debug ts=2026-04-21T00:05:00.761378532Z caller=checkpoint.go:378 msg="writing series" size="282 kB"
level=warn ts=2026-04-21T00:05:02.310195926Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:05:04.564020167Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
ts=2026-04-21T00:05:07.026941481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:05:07.309919963Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:05:08.366688854Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=debug ts=2026-04-21T00:05:12.170201097Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-21T00:05:12.309769385Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:05:15.973038128Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-21T00:05:17.309815704Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:05:19.775783288Z caller=checkpoint.go:378 msg="writing series" size="142 kB"
level=debug ts=2026-04-21T00:05:20.780998709Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:05:22.310348075Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:05:22.310366464Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:05:22.310387034Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:05:22.310410091Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:05:22.984065851Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T00:05:23.421190391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:05:23.429851603Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:05:23.578423184Z caller=checkpoint.go:378 msg="writing series" size="217 kB"
level=info ts=2026-04-21T00:05:27.057955193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:05:27.062143022Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-21T00:05:27.309949221Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:05:27.381352845Z caller=checkpoint.go:378 msg="writing series" size="135 kB"
level=debug ts=2026-04-21T00:05:27.381377253Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-04-21T00:05:27.92315819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:05:27.931257965Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-21T00:05:28.754573043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:40750"
level=debug ts=2026-04-21T00:05:31.184060944Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=warn ts=2026-04-21T00:05:32.309805784Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:05:34.986863017Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
ts=2026-04-21T00:05:37.028055347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:05:37.310326967Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:05:38.789752568Z caller=checkpoint.go:378 msg="writing series" size="272 kB"
level=warn ts=2026-04-21T00:05:42.310106401Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:05:42.592800392Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=debug ts=2026-04-21T00:05:46.395688449Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=warn ts=2026-04-21T00:05:47.310000337Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:05:48.436565823Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:05:50.198114854Z caller=checkpoint.go:378 msg="writing series" size="85 kB"
level=info ts=2026-04-21T00:05:52.287288766Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:05:52.288271567Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:05:52.309620474Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:05:52.309662118Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:05:52.309671251Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:05:52.309708393Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:05:54.000549858Z caller=checkpoint.go:378 msg="writing series" size="6.7 kB"
ts=2026-04-21T00:05:54.893035792Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:51380"
level=warn ts=2026-04-21T00:05:57.309550011Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:05:57.803419298Z caller=checkpoint.go:378 msg="writing series" size="329 kB"
level=debug ts=2026-04-21T00:06:01.610345185Z caller=checkpoint.go:378 msg="writing series" size="8.1 MB"
level=debug ts=2026-04-21T00:06:01.610370975Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.2 MB" series=9
level=warn ts=2026-04-21T00:06:02.309857897Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:06:05.409240986Z caller=checkpoint.go:378 msg="writing series" size="514 B"
ts=2026-04-21T00:06:07.030472887Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:06:07.309980408Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:06:09.212366847Z caller=checkpoint.go:378 msg="writing series" size="420 kB"
level=warn ts=2026-04-21T00:06:12.310413351Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:06:13.015009507Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=info ts=2026-04-21T00:06:13.478537353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:06:13.487526041Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:06:13.591751919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="70 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:06:13.598881792Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:06:16.81768163Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-21T00:06:17.310228409Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:06:18.444442079Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:06:20.62076008Z caller=checkpoint.go:378 msg="writing series" size="118 kB"
level=debug ts=2026-04-21T00:06:20.798022476Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T00:06:20.799666176Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T00:06:22.309980334Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:06:22.309980266Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:06:22.310009355Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:06:22.310018152Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T00:06:23.761651556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:06:23.770356097Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:06:23.770379943Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:06:23.770412101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:06:23.776752257Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:06:24.42388839Z caller=checkpoint.go:378 msg="writing series" size="142 kB"
level=warn ts=2026-04-21T00:06:27.310116386Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:06:28.226170856Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=debug ts=2026-04-21T00:06:32.02884Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-21T00:06:32.30969149Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:06:35.633686132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:06:35.643319385Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:06:35.831915646Z caller=checkpoint.go:378 msg="writing series" size="809 B"
ts=2026-04-21T00:06:37.03190303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:06:37.309829307Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:06:39.63496947Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=warn ts=2026-04-21T00:06:42.309835365Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:06:43.438344725Z caller=checkpoint.go:378 msg="writing series" size="841 kB"
level=debug ts=2026-04-21T00:06:43.4383717Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=11
level=debug ts=2026-04-21T00:06:44.474430487Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:06:47.240095508Z caller=checkpoint.go:378 msg="writing series" size="232 kB"
level=warn ts=2026-04-21T00:06:47.310218585Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:06:51.043054069Z caller=checkpoint.go:378 msg="writing series" size="421 kB"
level=info ts=2026-04-21T00:06:52.286537909Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:06:52.287229796Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:06:52.309680255Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:06:52.309681784Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:06:52.309704159Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:06:52.309712423Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:06:54.846367655Z caller=checkpoint.go:378 msg="writing series" size="306 kB"
level=info ts=2026-04-21T00:06:55.082844036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:06:55.089035969Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:06:56.357999764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="70 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:06:56.364916657Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T00:06:57.309776589Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:06:58.648233229Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=warn ts=2026-04-21T00:07:02.309694929Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:07:02.451053951Z caller=checkpoint.go:378 msg="writing series" size="51 kB"
level=debug ts=2026-04-21T00:07:02.451079476Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-04-21T00:07:03.746040759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:07:03.754115396Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:07:06.254098344Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
ts=2026-04-21T00:07:07.033592136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:07:07.310162544Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:07:10.057753054Z caller=checkpoint.go:378 msg="writing series" size="406 kB"
level=warn ts=2026-04-21T00:07:12.310097627Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:07:13.589561855Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:07:13.860002249Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=debug ts=2026-04-21T00:07:16.52458654Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T00:07:17.310338886Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:07:17.662972961Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=debug ts=2026-04-21T00:07:21.465802405Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=info ts=2026-04-21T00:07:22.309545013Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:07:22.309579846Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:07:22.309588864Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:07:22.309588493Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:07:25.268712292Z caller=checkpoint.go:378 msg="writing series" size="257 kB"
level=warn ts=2026-04-21T00:07:27.309570488Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:07:29.071235101Z caller=checkpoint.go:378 msg="writing series" size="266 kB"
level=debug ts=2026-04-21T00:07:29.071261046Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-04-21T00:07:32.309697937Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:07:32.873822824Z caller=checkpoint.go:378 msg="writing series" size="8.2 kB"
level=debug ts=2026-04-21T00:07:36.67644287Z caller=checkpoint.go:378 msg="writing series" size="88 kB"
ts=2026-04-21T00:07:37.035655873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:07:37.31007761Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:07:40.481577142Z caller=checkpoint.go:378 msg="writing series" size="2.3 MB"
level=debug ts=2026-04-21T00:07:40.481604057Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.4 MB" series=3
level=warn ts=2026-04-21T00:07:42.310281442Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:07:44.186705152Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:07:44.282199989Z caller=checkpoint.go:378 msg="writing series" size="328 kB"
level=info ts=2026-04-21T00:07:46.987453908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="70 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:07:46.999749745Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T00:07:47.310424354Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:07:48.085531498Z caller=checkpoint.go:378 msg="writing series" size="869 kB"
level=debug ts=2026-04-21T00:07:48.085556065Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-04-21T00:07:48.495274104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:07:48.49990112Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:07:49.980454683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:07:49.98856616Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:07:51.888219345Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-04-21T00:07:52.286647321Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:07:52.28677672Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:07:52.310340134Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:07:52.31036955Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:07:52.310377547Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:07:52.310394469Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:07:55.691303539Z caller=checkpoint.go:378 msg="writing series" size="336 kB"
level=warn ts=2026-04-21T00:07:57.309965102Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:07:58.598960622Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T00:07:59.493992287Z caller=checkpoint.go:378 msg="writing series" size="628 kB"
level=warn ts=2026-04-21T00:08:02.310339439Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:08:03.299177325Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
ts=2026-04-21T00:08:07.036901928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=debug ts=2026-04-21T00:08:07.099366773Z caller=checkpoint.go:378 msg="writing series" size="572 kB"
level=debug ts=2026-04-21T00:08:07.099394397Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=5
level=warn ts=2026-04-21T00:08:07.310390605Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:08:08.427896781Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:08:10.902674343Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=warn ts=2026-04-21T00:08:12.309531011Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:08:14.704911584Z caller=checkpoint.go:378 msg="writing series" size="188 kB"
level=warn ts=2026-04-21T00:08:17.309690303Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:08:18.507388394Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=info ts=2026-04-21T00:08:22.309720185Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:08:22.3097285Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:08:22.30975058Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:08:22.309761549Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:08:22.310829997Z caller=checkpoint.go:390 msg="flushing series" totalSize="227 kB" series=3
level=info ts=2026-04-21T00:08:22.315680531Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-04-21T00:08:22.316568955Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.006496113s
level=warn ts=2026-04-21T00:08:27.309684593Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:08:27.816028925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="70 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:08:27.826442792Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-21T00:08:28.840446558Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:34856"
level=warn ts=2026-04-21T00:08:32.309680471Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:08:36.128450925Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T00:08:37.038721036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:08:37.310161031Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T00:08:42.309674929Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T00:08:47.309677684Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:08:48.10525926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:08:48.110397642Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:08:49.93199778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:08:49.939455209Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T00:08:52.286596468Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:08:52.287048208Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:08:52.309493508Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-21T00:08:52.309511966Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:08:52.309540406Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:08:52.309549093Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:08:52.309574098Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:08:52.309872527Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=debug ts=2026-04-21T00:08:52.31005066Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
level=debug ts=2026-04-21T00:08:56.113636479Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-21T00:08:56.271959685Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T00:08:57.310383133Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:08:59.916184087Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-21T00:09:02.309793493Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:09:03.719544259Z caller=checkpoint.go:378 msg="writing series" size="245 kB"
ts=2026-04-21T00:09:07.040453304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:09:07.309930693Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:09:07.521825584Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=debug ts=2026-04-21T00:09:10.512210227Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:09:11.325254589Z caller=checkpoint.go:378 msg="writing series" size="709 kB"
level=debug ts=2026-04-21T00:09:11.325285056Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-04-21T00:09:12.310127971Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:09:12.461407935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="70 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:09:12.471964834Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:09:15.128131538Z caller=checkpoint.go:378 msg="writing series" size="1.0 MB"
level=warn ts=2026-04-21T00:09:17.31042649Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:09:18.9321555Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-04-21T00:09:18.932181986Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.2 MB" series=2
level=info ts=2026-04-21T00:09:22.310125295Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:09:22.310158142Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:09:22.310167103Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:09:22.310161121Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:09:22.732976558Z caller=checkpoint.go:378 msg="writing series" size="191 kB"
ts=2026-04-21T00:09:24.905874119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:57678"
level=debug ts=2026-04-21T00:09:26.535774163Z caller=checkpoint.go:378 msg="writing series" size="72 kB"
level=warn ts=2026-04-21T00:09:27.310142455Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:09:30.338827474Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=warn ts=2026-04-21T00:09:32.310202775Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:09:34.142065427Z caller=checkpoint.go:378 msg="writing series" size="106 kB"
level=info ts=2026-04-21T00:09:35.082665724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:09:35.086996963Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:09:35.087016471Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:09:35.087038615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:09:35.09028538Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:09:35.45085778Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T00:09:37.042999718Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:09:37.309965318Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:09:37.967575202Z caller=checkpoint.go:378 msg="writing series" size="23 MB"
level=debug ts=2026-04-21T00:09:37.96760277Z caller=checkpoint.go:390 msg="flushing series" totalSize="23 MB" series=5
level=debug ts=2026-04-21T00:09:41.74715237Z caller=checkpoint.go:378 msg="writing series" size="51 kB"
level=warn ts=2026-04-21T00:09:42.309534588Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:09:42.952762062Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:09:45.550617725Z caller=checkpoint.go:378 msg="writing series" size="1.0 MB"
level=debug ts=2026-04-21T00:09:45.550660091Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=2
level=warn ts=2026-04-21T00:09:47.30989335Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:09:47.609338966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:09:47.619502949Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T00:09:48.567686375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:09:48.571880969Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:09:49.353368027Z caller=checkpoint.go:378 msg="writing series" size="178 kB"
level=info ts=2026-04-21T00:09:52.286444124Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:09:52.286837777Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=warn ts=2026-04-21T00:09:52.310393647Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:09:52.310383878Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:09:52.310430469Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:09:52.310439052Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:09:53.155694693Z caller=checkpoint.go:378 msg="writing series" size="7.2 kB"
level=info ts=2026-04-21T00:09:53.360430577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="70 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:09:53.369316342Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:09:56.958713712Z caller=checkpoint.go:378 msg="writing series" size="77 kB"
level=warn ts=2026-04-21T00:09:57.309765284Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:09:58.770191299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:34640"
ts=2026-04-21T00:09:58.845968475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:55604"
level=debug ts=2026-04-21T00:10:00.761337257Z caller=checkpoint.go:378 msg="writing series" size="282 kB"
level=warn ts=2026-04-21T00:10:02.30969666Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:10:04.564323154Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
ts=2026-04-21T00:10:07.045485412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:10:07.310446605Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:10:08.367048165Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=debug ts=2026-04-21T00:10:12.169542389Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-21T00:10:12.310017311Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:10:15.972237796Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=debug ts=2026-04-21T00:10:16.761600773Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:10:17.309789289Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:10:19.775666643Z caller=checkpoint.go:378 msg="writing series" size="156 kB"
level=info ts=2026-04-21T00:10:22.309584492Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:10:22.309615528Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-21T00:10:22.309596976Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:10:22.309642143Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:10:23.578683827Z caller=checkpoint.go:378 msg="writing series" size="204 kB"
ts=2026-04-21T00:10:24.909887135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:32948"
level=info ts=2026-04-21T00:10:26.389851811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:10:26.39720348Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-21T00:10:27.310185893Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:10:27.381595319Z caller=checkpoint.go:378 msg="writing series" size="126 kB"
level=debug ts=2026-04-21T00:10:27.381612682Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=debug ts=2026-04-21T00:10:31.183816252Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=warn ts=2026-04-21T00:10:32.30953494Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:10:32.932168477Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T00:10:32.933979999Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-21T00:10:34.986940197Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
ts=2026-04-21T00:10:37.047971056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:10:37.310056551Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:10:38.789736365Z caller=checkpoint.go:378 msg="writing series" size="127 kB"
level=debug ts=2026-04-21T00:10:41.101682509Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:10:41.846256946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:10:41.854144617Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T00:10:41.884465344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="70 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:10:41.891721648Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:10:41.891742163Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:10:41.891767825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="70 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:10:41.898617595Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T00:10:42.309753716Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:10:42.592385162Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=debug ts=2026-04-21T00:10:46.395055261Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=warn ts=2026-04-21T00:10:47.310341222Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:10:50.197853792Z caller=checkpoint.go:378 msg="writing series" size="89 kB"
level=info ts=2026-04-21T00:10:52.286652968Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:10:52.287615131Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:10:52.310079235Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:10:52.310107551Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:10:52.310115919Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:10:52.310145975Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:10:53.905507467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="70 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:10:53.916275733Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:10:54.00053264Z caller=checkpoint.go:378 msg="writing series" size="6.7 kB"
level=warn ts=2026-04-21T00:10:57.309833432Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:10:57.803774792Z caller=checkpoint.go:378 msg="writing series" size="438 kB"
level=debug ts=2026-04-21T00:11:01.613003271Z caller=checkpoint.go:378 msg="writing series" size="8.7 MB"
level=debug ts=2026-04-21T00:11:01.61302884Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.7 MB" series=9
level=warn ts=2026-04-21T00:11:02.309832148Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:11:05.409107083Z caller=checkpoint.go:378 msg="writing series" size="642 B"
ts=2026-04-21T00:11:07.049817358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:11:07.310102221Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:11:09.212411532Z caller=checkpoint.go:378 msg="writing series" size="429 kB"
level=warn ts=2026-04-21T00:11:12.310369983Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:11:13.015233747Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=info ts=2026-04-21T00:11:14.467906404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:11:14.476118101Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:11:16.032733915Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:11:16.817149751Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-21T00:11:17.310229024Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:11:20.620758135Z caller=checkpoint.go:378 msg="writing series" size="124 kB"
level=info ts=2026-04-21T00:11:22.310401529Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:11:22.310433403Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:11:22.310442709Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:11:22.310444118Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:11:23.198862436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:11:23.203360276Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:11:24.423385146Z caller=checkpoint.go:378 msg="writing series" size="142 kB"
ts=2026-04-21T00:11:24.914065055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:36386"
level=warn ts=2026-04-21T00:11:27.310195118Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:11:28.225875044Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
ts=2026-04-21T00:11:28.851802497Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:52456"
level=info ts=2026-04-21T00:11:30.933428391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:11:30.94109811Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:11:32.029407739Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-21T00:11:32.31042762Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:11:32.466246859Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T00:11:35.831693686Z caller=checkpoint.go:378 msg="writing series" size="809 B"
ts=2026-04-21T00:11:37.051552065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:11:37.309579394Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:11:39.634199807Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=debug ts=2026-04-21T00:11:40.125641496Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:11:42.309609738Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:11:43.437169009Z caller=checkpoint.go:378 msg="writing series" size="841 kB"
level=debug ts=2026-04-21T00:11:43.437195562Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=11
level=debug ts=2026-04-21T00:11:47.240010251Z caller=checkpoint.go:378 msg="writing series" size="256 kB"
level=warn ts=2026-04-21T00:11:47.310109041Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:11:51.043649419Z caller=checkpoint.go:378 msg="writing series" size="515 kB"
level=info ts=2026-04-21T00:11:52.286781169Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:11:52.28788299Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:11:52.310374153Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:11:52.310387636Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:11:52.31040612Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:11:52.310432916Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:11:54.845433934Z caller=checkpoint.go:378 msg="writing series" size="324 kB"
level=debug ts=2026-04-21T00:11:54.845460409Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=3
level=warn ts=2026-04-21T00:11:57.310372254Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:11:58.648388344Z caller=checkpoint.go:378 msg="writing series" size="66 kB"
level=warn ts=2026-04-21T00:12:02.310179893Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:12:02.451534309Z caller=checkpoint.go:378 msg="writing series" size="51 kB"
level=info ts=2026-04-21T00:12:03.134593081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:12:03.142658815Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:12:05.552763863Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:12:06.254696809Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-04-21T00:12:06.848658108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:12:06.85311058Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-04-21T00:12:07.053328622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:12:07.309676699Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:12:10.05726429Z caller=checkpoint.go:378 msg="writing series" size="404 kB"
level=warn ts=2026-04-21T00:12:12.310326471Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:12:13.859808849Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=info ts=2026-04-21T00:12:15.496679167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:12:15.506201107Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-21T00:12:17.310355557Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:12:17.663127734Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=debug ts=2026-04-21T00:12:21.465024703Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=info ts=2026-04-21T00:12:22.310286374Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:12:22.310301475Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:12:22.310315395Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:12:22.310323769Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:12:25.268745781Z caller=checkpoint.go:378 msg="writing series" size="257 kB"
level=warn ts=2026-04-21T00:12:27.310089324Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:12:28.855587972Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:50342"
level=debug ts=2026-04-21T00:12:29.071370653Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=debug ts=2026-04-21T00:12:29.07139686Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-04-21T00:12:32.310416917Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:12:32.874014002Z caller=checkpoint.go:378 msg="writing series" size="8.2 kB"
level=debug ts=2026-04-21T00:12:34.014384044Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T00:12:36.677186951Z caller=checkpoint.go:378 msg="writing series" size="90 kB"
ts=2026-04-21T00:12:37.055853499Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=debug ts=2026-04-21T00:12:37.117112592Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:12:37.310149816Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:12:40.481753236Z caller=checkpoint.go:378 msg="writing series" size="2.4 MB"
level=debug ts=2026-04-21T00:12:40.481781846Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.5 MB" series=3
level=warn ts=2026-04-21T00:12:42.309533368Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:12:44.283093233Z caller=checkpoint.go:378 msg="writing series" size="430 kB"
level=warn ts=2026-04-21T00:12:47.310241165Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:12:48.086181705Z caller=checkpoint.go:378 msg="writing series" size="869 kB"
level=debug ts=2026-04-21T00:12:48.086206192Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=2
level=debug ts=2026-04-21T00:12:51.888065724Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=info ts=2026-04-21T00:12:52.287073372Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:12:52.288037769Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:12:52.310428955Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:12:52.310439043Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:12:52.310454711Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:12:52.31046292Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:12:55.690674051Z caller=checkpoint.go:378 msg="writing series" size="336 kB"
level=info ts=2026-04-21T00:12:56.877777282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:12:56.884107914Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-21T00:12:57.309948904Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:12:58.781894864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:51720"
level=debug ts=2026-04-21T00:12:59.493944109Z caller=checkpoint.go:378 msg="writing series" size="649 kB"
level=info ts=2026-04-21T00:13:00.077771846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:13:00.084873117Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:13:00.084895376Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:13:00.084925109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:13:00.093185897Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T00:13:00.608928514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:13:00.616267762Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T00:13:02.310180897Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:13:03.296358514Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=debug ts=2026-04-21T00:13:04.267752777Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T00:13:07.057891126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=debug ts=2026-04-21T00:13:07.099268179Z caller=checkpoint.go:378 msg="writing series" size="572 kB"
level=debug ts=2026-04-21T00:13:07.099293395Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=5
level=warn ts=2026-04-21T00:13:07.309746803Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:13:10.902284934Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=warn ts=2026-04-21T00:13:12.310321819Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:13:14.704967949Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=warn ts=2026-04-21T00:13:17.309897266Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:13:17.661410854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:13:17.678976444Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:13:18.50750669Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=debug ts=2026-04-21T00:13:22.310440713Z caller=checkpoint.go:390 msg="flushing series" totalSize="228 kB" series=3
level=info ts=2026-04-21T00:13:22.31044559Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:13:22.31044855Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:13:22.31046791Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:13:22.310477233Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T00:13:22.31656914Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-04-21T00:13:22.320062969Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.010062794s
level=debug ts=2026-04-21T00:13:24.152572867Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T00:13:27.309713003Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:13:28.859596702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:43844"
level=warn ts=2026-04-21T00:13:32.310414796Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:13:34.771278481Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T00:13:37.059150511Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:13:37.310407317Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T00:13:42.309752615Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:13:43.763009766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:13:43.771424923Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:13:44.347305259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:13:44.35590712Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T00:13:47.310263146Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:13:50.979835728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:13:50.987864613Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:13:52.286370246Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:13:52.286430032Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T00:13:52.309946251Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:13:52.309977034Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:13:52.309970478Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-21T00:13:52.309985775Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:13:52.310003657Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:13:52.310111779Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=debug ts=2026-04-21T00:13:52.310298026Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
level=debug ts=2026-04-21T00:13:56.113757015Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-21T00:13:57.309907429Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:13:58.784851599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:55560"
level=debug ts=2026-04-21T00:13:59.916125775Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=debug ts=2026-04-21T00:14:01.554979153Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:14:02.310236057Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:14:03.719583099Z caller=checkpoint.go:378 msg="writing series" size="302 kB"
ts=2026-04-21T00:14:07.060904789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:14:07.310453498Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:14:07.521686093Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
level=debug ts=2026-04-21T00:14:11.32520462Z caller=checkpoint.go:378 msg="writing series" size="532 kB"
level=warn ts=2026-04-21T00:14:12.30982366Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:14:15.128765201Z caller=checkpoint.go:378 msg="writing series" size="1.0 MB"
level=debug ts=2026-04-21T00:14:15.128790217Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.0 MB" series=7
level=warn ts=2026-04-21T00:14:17.310064742Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:14:18.931500687Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-04-21T00:14:18.931524229Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=1
level=info ts=2026-04-21T00:14:22.310287199Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:14:22.310321446Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:14:22.310330062Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:14:22.310322166Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:14:22.733194865Z caller=checkpoint.go:378 msg="writing series" size="193 kB"
level=debug ts=2026-04-21T00:14:24.034372785Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T00:14:24.925909134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:43942"
level=debug ts=2026-04-21T00:14:26.536120687Z caller=checkpoint.go:378 msg="writing series" size="80 kB"
level=warn ts=2026-04-21T00:14:27.309855942Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:14:28.862546285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:48756"
level=debug ts=2026-04-21T00:14:30.338860159Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=warn ts=2026-04-21T00:14:32.310208619Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:14:32.598821117Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:14:33.843861718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:14:33.850039417Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:14:34.142213494Z caller=checkpoint.go:378 msg="writing series" size="114 kB"
ts=2026-04-21T00:14:37.06254013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:14:37.309539832Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:14:37.963017632Z caller=checkpoint.go:378 msg="writing series" size="23 MB"
level=debug ts=2026-04-21T00:14:37.963050619Z caller=checkpoint.go:390 msg="flushing series" totalSize="23 MB" series=5
level=info ts=2026-04-21T00:14:40.198708173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:14:40.215517428Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:14:41.747103894Z caller=checkpoint.go:378 msg="writing series" size="51 kB"
level=warn ts=2026-04-21T00:14:42.309537139Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:14:45.551454056Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-04-21T00:14:45.551477895Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=2
level=warn ts=2026-04-21T00:14:47.309980827Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:14:48.307002202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:14:48.31536957Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:14:49.353834985Z caller=checkpoint.go:378 msg="writing series" size="208 kB"
level=info ts=2026-04-21T00:14:52.286516453Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:14:52.286526867Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:14:52.310104111Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:14:52.310135968Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-21T00:14:52.310126712Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:14:52.310144654Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:14:53.156192849Z caller=checkpoint.go:378 msg="writing series" size="7.2 kB"
level=debug ts=2026-04-21T00:14:56.959101884Z caller=checkpoint.go:378 msg="writing series" size="77 kB"
level=warn ts=2026-04-21T00:14:57.309896802Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:14:58.789467331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:60874"
ts=2026-04-21T00:14:58.864888612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:37874"
level=debug ts=2026-04-21T00:15:00.762082119Z caller=checkpoint.go:378 msg="writing series" size="282 kB"
level=warn ts=2026-04-21T00:15:02.310199458Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:15:03.830019873Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:15:03.830094388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:15:03.839546695Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T00:15:04.564722415Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
ts=2026-04-21T00:15:07.063795614Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:15:07.309996096Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:15:08.367646855Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=debug ts=2026-04-21T00:15:09.45045Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T00:15:09.45218663Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-21T00:15:12.170031017Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-21T00:15:12.310438932Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:15:15.972678101Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-21T00:15:16.81634063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:15:16.82544753Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T00:15:16.957878537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:15:16.961923563Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-21T00:15:17.309783315Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:15:19.775455976Z caller=checkpoint.go:378 msg="writing series" size="174 kB"
level=info ts=2026-04-21T00:15:22.309753115Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:15:22.309765893Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:15:22.30978962Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:15:22.309799316Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:15:23.577998194Z caller=checkpoint.go:378 msg="writing series" size="231 kB"
level=warn ts=2026-04-21T00:15:27.309954613Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:15:27.381329845Z caller=checkpoint.go:378 msg="writing series" size="172 kB"
level=debug ts=2026-04-21T00:15:27.381351685Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=11
ts=2026-04-21T00:15:28.790959697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43738"
ts=2026-04-21T00:15:28.866747109Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:55158"
level=debug ts=2026-04-21T00:15:28.999937649Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:15:31.184314066Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=warn ts=2026-04-21T00:15:32.310421461Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:15:32.696895362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:15:32.705149267Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:15:34.987290864Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=info ts=2026-04-21T00:15:35.218230192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:15:35.22738507Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-21T00:15:37.06602092Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:15:37.3102648Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:15:38.789934024Z caller=checkpoint.go:378 msg="writing series" size="194 kB"
level=info ts=2026-04-21T00:15:39.199299044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:15:39.205747482Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T00:15:42.310294905Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:15:42.593015078Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=debug ts=2026-04-21T00:15:46.395985597Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=warn ts=2026-04-21T00:15:47.309712968Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:15:50.198132854Z caller=checkpoint.go:378 msg="writing series" size="89 kB"
level=debug ts=2026-04-21T00:15:51.781302976Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T00:15:52.286567778Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:15:52.286582649Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=debug ts=2026-04-21T00:15:52.287886799Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-21T00:15:52.287953859Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974144 n=1
level=debug ts=2026-04-21T00:15:52.28801892Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974144 n=1
level=info ts=2026-04-21T00:15:52.310174251Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:15:52.310210073Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:15:52.310219004Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:15:52.310236058Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:15:54.001288867Z caller=checkpoint.go:378 msg="writing series" size="6.7 kB"
level=warn ts=2026-04-21T00:15:57.310200219Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:15:57.804124375Z caller=checkpoint.go:378 msg="writing series" size="344 kB"
ts=2026-04-21T00:15:58.868274246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:56924"
level=debug ts=2026-04-21T00:16:01.612388834Z caller=checkpoint.go:378 msg="writing series" size="9.2 MB"
level=debug ts=2026-04-21T00:16:01.612413544Z caller=checkpoint.go:390 msg="flushing series" totalSize="10 MB" series=9
level=warn ts=2026-04-21T00:16:02.30960358Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:16:03.707725857Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:16:05.409896632Z caller=checkpoint.go:378 msg="writing series" size="657 B"
level=info ts=2026-04-21T00:16:05.602534487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:16:05.61336031Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:16:05.613387129Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:16:05.613415181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:16:05.617238594Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-04-21T00:16:07.068073927Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:16:07.309903686Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:16:09.212364586Z caller=checkpoint.go:378 msg="writing series" size="443 kB"
level=warn ts=2026-04-21T00:16:12.310449761Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:16:13.014869731Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=info ts=2026-04-21T00:16:16.033259438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:16:16.039803182Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:16:16.818180681Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-21T00:16:17.310404883Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:16:18.007850039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:16:18.016139484Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:16:20.620447959Z caller=checkpoint.go:378 msg="writing series" size="131 kB"
level=info ts=2026-04-21T00:16:22.310359383Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:16:22.310389418Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:16:22.310398031Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:16:22.310419179Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:16:24.423171407Z caller=checkpoint.go:378 msg="writing series" size="142 kB"
level=debug ts=2026-04-21T00:16:26.594575399Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T00:16:27.310595841Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:16:28.226654532Z caller=checkpoint.go:378 msg="writing series" size="39 kB"
ts=2026-04-21T00:16:28.795170545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:42156"
level=debug ts=2026-04-21T00:16:29.443332529Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:16:30.436680283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:16:30.447586716Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T00:16:30.455592893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:16:30.462578935Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T00:16:32.028653362Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-21T00:16:32.309505968Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:16:35.832197738Z caller=checkpoint.go:378 msg="writing series" size="809 B"
ts=2026-04-21T00:16:37.070294118Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:16:37.309761688Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:16:39.634564872Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=warn ts=2026-04-21T00:16:42.309565424Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:16:43.438298608Z caller=checkpoint.go:378 msg="writing series" size="841 kB"
level=debug ts=2026-04-21T00:16:43.438327515Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=11
level=debug ts=2026-04-21T00:16:47.240329158Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=warn ts=2026-04-21T00:16:47.309802471Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:16:51.043589317Z caller=checkpoint.go:378 msg="writing series" size="377 kB"
level=info ts=2026-04-21T00:16:52.181826237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:16:52.189140967Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:16:52.286817784Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:16:52.287305023Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:16:52.309706329Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:16:52.309746445Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:16:52.309755069Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:16:52.309748921Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:16:54.719742688Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:16:54.719792248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="489 kB" avg_comp="489 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:16:54.724026847Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T00:16:54.846684919Z caller=checkpoint.go:378 msg="writing series" size="435 kB"
level=warn ts=2026-04-21T00:16:57.309515916Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:16:58.649364598Z caller=checkpoint.go:378 msg="writing series" size="68 kB"
ts=2026-04-21T00:16:58.797260464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56172"
level=info ts=2026-04-21T00:16:59.863375854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:16:59.870434078Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-21T00:17:02.309726067Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:17:02.452122322Z caller=checkpoint.go:378 msg="writing series" size="51 kB"
level=info ts=2026-04-21T00:17:05.302777046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:17:05.307466447Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T00:17:06.254834665Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
ts=2026-04-21T00:17:07.072597756Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:17:07.310569934Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:17:09.455147084Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T00:17:10.057137586Z caller=checkpoint.go:378 msg="writing series" size="417 kB"
level=debug ts=2026-04-21T00:17:10.057171152Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=7
level=warn ts=2026-04-21T00:17:12.310090142Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:17:13.860100738Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=info ts=2026-04-21T00:17:15.682311624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:17:15.691397734Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:17:15.691430459Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:17:15.691460153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:17:15.697951039Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T00:17:17.310093731Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:17:17.662691461Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=debug ts=2026-04-21T00:17:21.465989456Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=debug ts=2026-04-21T00:17:22.189333694Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:17:22.310378874Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:17:22.310398147Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:17:22.310413553Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:17:22.310422222Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:17:25.268647444Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=info ts=2026-04-21T00:17:26.804134486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:17:26.815039762Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T00:17:27.310059999Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:17:28.189597151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:17:28.193967242Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-21T00:17:28.799360058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:50302"
ts=2026-04-21T00:17:28.873791234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:54026"
level=debug ts=2026-04-21T00:17:29.071907845Z caller=checkpoint.go:378 msg="writing series" size="144 kB"
level=info ts=2026-04-21T00:17:30.321328078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:17:30.328951536Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T00:17:32.30957537Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:17:32.874069776Z caller=checkpoint.go:378 msg="writing series" size="8.2 kB"
level=debug ts=2026-04-21T00:17:36.676672286Z caller=checkpoint.go:378 msg="writing series" size="94 kB"
ts=2026-04-21T00:17:37.074196275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:17:37.31043199Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:17:40.482535486Z caller=checkpoint.go:378 msg="writing series" size="2.4 MB"
level=debug ts=2026-04-21T00:17:40.482559513Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.0 MB" series=8
level=info ts=2026-04-21T00:17:41.243352956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:17:41.249585675Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-21T00:17:42.310487265Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:17:44.282757785Z caller=checkpoint.go:378 msg="writing series" size="298 kB"
level=warn ts=2026-04-21T00:17:47.31051291Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:17:48.085968113Z caller=checkpoint.go:378 msg="writing series" size="869 kB"
level=debug ts=2026-04-21T00:17:48.085992707Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=2
level=debug ts=2026-04-21T00:17:51.888709497Z caller=checkpoint.go:378 msg="writing series" size="3.7 kB"
level=info ts=2026-04-21T00:17:52.286853315Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:17:52.287952228Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:17:52.310394237Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:17:52.31041809Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:17:52.310426602Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:17:52.310444271Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:17:55.51167787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:17:55.521357797Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:17:55.69127714Z caller=checkpoint.go:378 msg="writing series" size="336 kB"
level=warn ts=2026-04-21T00:17:57.310120327Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:17:57.879960507Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T00:17:58.875357838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:51928"
level=debug ts=2026-04-21T00:17:59.494201086Z caller=checkpoint.go:378 msg="writing series" size="760 kB"
level=debug ts=2026-04-21T00:17:59.494232007Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=3
level=warn ts=2026-04-21T00:18:02.309863546Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:18:03.29720626Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-04-21T00:18:06.151326648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:18:06.160316489Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-21T00:18:07.077571324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=debug ts=2026-04-21T00:18:07.099885745Z caller=checkpoint.go:378 msg="writing series" size="558 kB"
level=warn ts=2026-04-21T00:18:07.310525867Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:18:07.497708787Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T00:18:10.902825269Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=warn ts=2026-04-21T00:18:12.310036389Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:18:13.325119813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:18:13.331572224Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T00:18:14.704996222Z caller=checkpoint.go:378 msg="writing series" size="190 kB"
level=warn ts=2026-04-21T00:18:17.310218202Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:18:18.507921785Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=info ts=2026-04-21T00:18:22.31040754Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:18:22.310407463Z caller=checkpoint.go:390 msg="flushing series" totalSize="790 kB" series=5
level=debug ts=2026-04-21T00:18:22.310437105Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:18:22.310446218Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:18:22.310444727Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:18:22.317802415Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-04-21T00:18:22.321434839Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.01118864s
level=info ts=2026-04-21T00:18:22.760846325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:18:22.76547875Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T00:18:25.927140347Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:18:27.310048821Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:18:28.803089885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43492"
ts=2026-04-21T00:18:28.877658454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:41068"
level=info ts=2026-04-21T00:18:30.221550925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:18:30.226186326Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-21T00:18:32.309635096Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:18:37.079775847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:18:37.310122399Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T00:18:42.310504601Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T00:18:47.310058063Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:18:52.286840541Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:18:52.287739596Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:18:52.310065879Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-21T00:18:52.310108683Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-04-21T00:18:52.310101758Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:18:52.310094758Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:18:52.310141932Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T00:18:52.31024284Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=debug ts=2026-04-21T00:18:52.31045538Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
level=debug ts=2026-04-21T00:18:52.905309752Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:18:53.043591814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:18:53.053592066Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:18:56.114025289Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=info ts=2026-04-21T00:18:56.392984307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:18:56.400573974Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T00:18:57.310368256Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:18:59.150137241Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T00:18:59.151744551Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-21T00:18:59.916683262Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=info ts=2026-04-21T00:19:02.276506644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:19:02.283709079Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T00:19:02.310066083Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:19:03.71958427Z caller=checkpoint.go:378 msg="writing series" size="106 kB"
ts=2026-04-21T00:19:07.081522956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=info ts=2026-04-21T00:19:07.241371809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:19:07.246021273Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T00:19:07.310134292Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:19:07.52211075Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=debug ts=2026-04-21T00:19:11.325826926Z caller=checkpoint.go:378 msg="writing series" size="732 kB"
level=warn ts=2026-04-21T00:19:12.310273974Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:19:15.129018349Z caller=checkpoint.go:378 msg="writing series" size="1.0 MB"
level=debug ts=2026-04-21T00:19:15.129044533Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.0 MB" series=7
level=debug ts=2026-04-21T00:19:17.143136285Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:19:17.310581013Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:19:18.932614595Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-21T00:19:18.932660175Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-04-21T00:19:22.310052096Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:19:22.310081823Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:19:22.310090845Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:19:22.310111167Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:19:22.733917608Z caller=checkpoint.go:378 msg="writing series" size="196 kB"
level=debug ts=2026-04-21T00:19:26.536661759Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=warn ts=2026-04-21T00:19:27.310361795Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:19:28.806173955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:33004"
level=info ts=2026-04-21T00:19:29.439519035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:19:29.446399037Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:19:30.339044571Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=warn ts=2026-04-21T00:19:32.309908016Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:19:34.142381191Z caller=checkpoint.go:378 msg="writing series" size="123 kB"
level=info ts=2026-04-21T00:19:34.949968051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:19:34.957493507Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:19:34.957518232Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:19:34.957549897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:19:34.964287924Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-21T00:19:37.083174808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:19:37.310360355Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:19:37.966875541Z caller=checkpoint.go:378 msg="writing series" size="23 MB"
level=debug ts=2026-04-21T00:19:37.966902971Z caller=checkpoint.go:390 msg="flushing series" totalSize="23 MB" series=5
level=debug ts=2026-04-21T00:19:41.747284241Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=warn ts=2026-04-21T00:19:42.309867063Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:19:45.213364632Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T00:19:45.552351742Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-21T00:19:45.552383056Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=2
level=info ts=2026-04-21T00:19:45.843650892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:19:45.853126077Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-21T00:19:47.310173365Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:19:49.353542801Z caller=checkpoint.go:378 msg="writing series" size="239 kB"
level=info ts=2026-04-21T00:19:49.895751079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:19:49.903807329Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:19:50.651964813Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:19:52.286568676Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:19:52.287003724Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:19:52.31041325Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:19:52.310448105Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:19:52.310456975Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:19:52.310446275Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:19:53.156075463Z caller=checkpoint.go:378 msg="writing series" size="7.2 kB"
level=debug ts=2026-04-21T00:19:56.959305246Z caller=checkpoint.go:378 msg="writing series" size="77 kB"
level=warn ts=2026-04-21T00:19:57.3099134Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:19:58.634858103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:19:58.643398036Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T00:19:59.041472294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:19:59.046077684Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T00:20:00.761424165Z caller=checkpoint.go:378 msg="writing series" size="282 kB"
level=warn ts=2026-04-21T00:20:02.309961065Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:20:04.564616031Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
ts=2026-04-21T00:20:07.084948409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:20:07.310186021Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:20:08.367104047Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=debug ts=2026-04-21T00:20:12.170347835Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-21T00:20:12.309756064Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:20:12.341913385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:20:12.349230853Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:20:15.973143271Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-21T00:20:17.309895083Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:20:18.045508618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:20:18.053547448Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:20:19.776010173Z caller=checkpoint.go:378 msg="writing series" size="187 kB"
level=debug ts=2026-04-21T00:20:21.759181869Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:20:22.310031539Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:20:22.310031532Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:20:22.310064252Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:20:22.310074838Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:20:23.578698882Z caller=checkpoint.go:378 msg="writing series" size="219 kB"
level=debug ts=2026-04-21T00:20:23.578722075Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
ts=2026-04-21T00:20:24.948664737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:33366"
level=debug ts=2026-04-21T00:20:25.808554966Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T00:20:27.310437159Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:20:27.382071347Z caller=checkpoint.go:378 msg="writing series" size="269 kB"
ts=2026-04-21T00:20:28.809903099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:36268"
ts=2026-04-21T00:20:28.883808801Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:36226"
level=debug ts=2026-04-21T00:20:31.183934243Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=warn ts=2026-04-21T00:20:32.310292573Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:20:34.987381507Z caller=checkpoint.go:378 msg="writing series" size="6.1 kB"
ts=2026-04-21T00:20:37.087271876Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:20:37.310069462Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:20:38.79041224Z caller=checkpoint.go:378 msg="writing series" size="248 kB"
level=info ts=2026-04-21T00:20:41.343750546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:20:41.354691982Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T00:20:42.309522026Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:20:42.59322205Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=info ts=2026-04-21T00:20:44.227313692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:20:44.234751738Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T00:20:46.395976973Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=info ts=2026-04-21T00:20:46.92345464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:20:46.932230283Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T00:20:47.310338664Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:20:47.782830408Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:20:50.1982866Z caller=checkpoint.go:378 msg="writing series" size="104 kB"
level=info ts=2026-04-21T00:20:52.286386683Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:20:52.286859828Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:20:52.310239111Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:20:52.310269324Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:20:52.310278262Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:20:52.31027391Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:20:54.000687856Z caller=checkpoint.go:378 msg="writing series" size="7.8 kB"
level=info ts=2026-04-21T00:20:56.650371789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:20:56.655955344Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-21T00:20:57.310383839Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:20:57.806098342Z caller=checkpoint.go:378 msg="writing series" size="485 kB"
level=debug ts=2026-04-21T00:20:57.806131676Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=9
ts=2026-04-21T00:20:58.88591863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:34854"
level=debug ts=2026-04-21T00:21:01.613864764Z caller=checkpoint.go:378 msg="writing series" size="10 MB"
level=debug ts=2026-04-21T00:21:01.613890815Z caller=checkpoint.go:390 msg="flushing series" totalSize="10 MB" series=1
level=warn ts=2026-04-21T00:21:02.309568623Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:21:05.409267649Z caller=checkpoint.go:378 msg="writing series" size="657 B"
ts=2026-04-21T00:21:07.088853471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:21:07.310048357Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:21:09.212315882Z caller=checkpoint.go:378 msg="writing series" size="463 kB"
level=info ts=2026-04-21T00:21:10.253039487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:21:10.262951143Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-21T00:21:12.30979249Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:21:13.015241276Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=debug ts=2026-04-21T00:21:16.818543999Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-21T00:21:17.309831699Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:21:20.620860242Z caller=checkpoint.go:378 msg="writing series" size="137 kB"
level=info ts=2026-04-21T00:21:22.309693241Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:21:22.30973591Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-21T00:21:22.309724318Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:21:22.309744752Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:21:22.574307506Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:21:24.364773807Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T00:21:24.424144749Z caller=checkpoint.go:378 msg="writing series" size="142 kB"
level=info ts=2026-04-21T00:21:25.399961206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:21:25.407930092Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T00:21:27.309907796Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:21:28.226904501Z caller=checkpoint.go:378 msg="writing series" size="40 kB"
ts=2026-04-21T00:21:28.888301569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:42652"
level=debug ts=2026-04-21T00:21:32.029692883Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-21T00:21:32.309806865Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:21:35.831847524Z caller=checkpoint.go:378 msg="writing series" size="809 B"
ts=2026-04-21T00:21:37.089976153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:21:37.31016325Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:21:38.569320431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:21:38.576714541Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:21:39.634966809Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=warn ts=2026-04-21T00:21:42.310528333Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:21:42.575745601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:21:42.582569219Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:21:42.582593923Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:21:42.582638982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:21:42.588567343Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T00:21:43.428202146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:21:43.432999368Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T00:21:43.438799942Z caller=checkpoint.go:378 msg="writing series" size="841 kB"
level=debug ts=2026-04-21T00:21:43.438819882Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=11
level=debug ts=2026-04-21T00:21:47.240391271Z caller=checkpoint.go:378 msg="writing series" size="74 kB"
level=warn ts=2026-04-21T00:21:47.310260358Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:21:51.044238278Z caller=checkpoint.go:378 msg="writing series" size="462 kB"
level=debug ts=2026-04-21T00:21:52.286160366Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:21:52.287214465Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T00:21:52.309542822Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:21:52.309563288Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:21:52.309588304Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:21:52.309597769Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:21:54.402072824Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:21:54.845868663Z caller=checkpoint.go:378 msg="writing series" size="273 kB"
level=warn ts=2026-04-21T00:21:57.309997017Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:21:58.649360734Z caller=checkpoint.go:378 msg="writing series" size="74 kB"
level=info ts=2026-04-21T00:21:58.68374381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:21:58.691918128Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-21T00:21:58.890419199Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:37420"
level=debug ts=2026-04-21T00:22:01.918706904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T00:22:02.309675322Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:22:02.452082671Z caller=checkpoint.go:378 msg="writing series" size="51 kB"
level=debug ts=2026-04-21T00:22:06.254178832Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
ts=2026-04-21T00:22:07.091486901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=info ts=2026-04-21T00:22:07.260658736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:22:07.267762772Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-21T00:22:07.310090525Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:22:09.927914956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:22:09.934846364Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:22:10.057319644Z caller=checkpoint.go:378 msg="writing series" size="417 kB"
level=debug ts=2026-04-21T00:22:10.057343955Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=7
level=warn ts=2026-04-21T00:22:12.310346848Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:22:13.860518737Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=warn ts=2026-04-21T00:22:17.309760105Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:22:17.662849759Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=info ts=2026-04-21T00:22:18.640532459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:22:18.645792972Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:22:18.64581706Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:22:18.64584219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:22:18.649550977Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:22:21.369243253Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:22:21.465452315Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-21T00:22:22.309811771Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:22:22.309820947Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:22:22.309848185Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:22:22.309856881Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T00:22:22.586550263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:22:22.593260287Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T00:22:25.268565405Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=warn ts=2026-04-21T00:22:27.310314222Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:22:28.53123598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:22:28.536495546Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-21T00:22:28.816248014Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:57978"
level=debug ts=2026-04-21T00:22:29.071440403Z caller=checkpoint.go:378 msg="writing series" size="198 kB"
level=info ts=2026-04-21T00:22:29.129750912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:22:29.135716052Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-21T00:22:32.310194738Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:22:32.873871206Z caller=checkpoint.go:378 msg="writing series" size="8.2 kB"
level=debug ts=2026-04-21T00:22:36.486602172Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T00:22:36.488291193Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-21T00:22:36.676753517Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
ts=2026-04-21T00:22:37.092560859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:22:37.310534259Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:22:40.481258755Z caller=checkpoint.go:378 msg="writing series" size="2.5 MB"
level=debug ts=2026-04-21T00:22:40.481284606Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.1 MB" series=8
level=warn ts=2026-04-21T00:22:42.310101247Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:22:44.283047507Z caller=checkpoint.go:378 msg="writing series" size="415 kB"
level=warn ts=2026-04-21T00:22:47.310391796Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:22:48.086261884Z caller=checkpoint.go:378 msg="writing series" size="869 kB"
level=debug ts=2026-04-21T00:22:48.086287797Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=2
level=debug ts=2026-04-21T00:22:49.651203032Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:22:51.888101284Z caller=checkpoint.go:378 msg="writing series" size="4.0 kB"
level=info ts=2026-04-21T00:22:51.975183323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:22:51.984051259Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T00:22:52.286743024Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:22:52.287221236Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:22:52.309504091Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:22:52.309520571Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:22:52.309537289Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:22:52.309547462Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:22:55.691334716Z caller=checkpoint.go:378 msg="writing series" size="336 kB"
level=warn ts=2026-04-21T00:22:57.30968923Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:22:58.47915916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:22:58.486771165Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-21T00:22:58.818690777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:51018"
level=debug ts=2026-04-21T00:22:59.494704012Z caller=checkpoint.go:378 msg="writing series" size="755 kB"
level=debug ts=2026-04-21T00:22:59.494727652Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-04-21T00:23:00.886919001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:23:00.892995218Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-21T00:23:02.310446367Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:23:03.297073922Z caller=checkpoint.go:378 msg="writing series" size="5.5 kB"
ts=2026-04-21T00:23:07.094435518Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=debug ts=2026-04-21T00:23:07.099680821Z caller=checkpoint.go:378 msg="writing series" size="558 kB"
level=warn ts=2026-04-21T00:23:07.310457977Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:23:08.095190542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:23:08.10183068Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T00:23:10.902665875Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=warn ts=2026-04-21T00:23:12.309672208Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:23:14.705078846Z caller=checkpoint.go:378 msg="writing series" size="190 kB"
level=warn ts=2026-04-21T00:23:17.309656651Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:23:18.507990105Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=debug ts=2026-04-21T00:23:21.253364296Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:23:22.310244552Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:23:22.310294022Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:23:22.310302798Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:23:22.310321421Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:23:22.311353699Z caller=checkpoint.go:390 msg="flushing series" totalSize="792 kB" series=5
level=info ts=2026-04-21T00:23:22.317461919Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-04-21T00:23:22.321092664Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.010696331s
level=info ts=2026-04-21T00:23:24.447371331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:23:24.452951492Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:23:24.45297407Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:23:24.452997964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:23:24.456594227Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-21T00:23:24.960041726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:48774"
level=warn ts=2026-04-21T00:23:27.309756475Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:23:28.820479052Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:57760"
level=debug ts=2026-04-21T00:23:31.556176934Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T00:23:32.310388366Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:23:35.933126663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:23:35.942308366Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-21T00:23:37.097070311Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:23:37.309832366Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:23:40.049346521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:23:40.053425617Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T00:23:42.309720055Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:23:46.586032793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:23:46.593667771Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:23:46.593696249Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:23:46.593725971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:23:46.600123712Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T00:23:47.309663651Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:23:52.286563129Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:23:52.2875982Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:23:52.309949681Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-21T00:23:52.309968046Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:23:52.309972425Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:23:52.309991265Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:23:52.310000072Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T00:23:52.310158688Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=debug ts=2026-04-21T00:23:52.310384752Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
level=info ts=2026-04-21T00:23:53.831846799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:23:53.836695331Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:23:54.387610935Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T00:23:54.961598624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:50378"
level=debug ts=2026-04-21T00:23:56.11390344Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-21T00:23:57.310537167Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:23:58.822151587Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56216"
level=info ts=2026-04-21T00:23:58.993593143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:23:59.00068586Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:23:59.916409363Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-21T00:24:02.309569345Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:24:03.719938654Z caller=checkpoint.go:378 msg="writing series" size="198 kB"
ts=2026-04-21T00:24:07.098792777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:24:07.310294534Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:24:07.363396668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:24:07.371710487Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T00:24:07.522070567Z caller=checkpoint.go:378 msg="writing series" size="51 kB"
level=info ts=2026-04-21T00:24:10.228851594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:24:10.233508596Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T00:24:11.325137963Z caller=checkpoint.go:378 msg="writing series" size="688 kB"
level=warn ts=2026-04-21T00:24:12.309572272Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:24:15.128167738Z caller=checkpoint.go:378 msg="writing series" size="912 kB"
level=debug ts=2026-04-21T00:24:15.128192419Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=7
level=warn ts=2026-04-21T00:24:17.309714734Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:24:18.931106173Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-04-21T00:24:18.931131757Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=1
level=info ts=2026-04-21T00:24:20.626675203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:24:20.635988036Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T00:24:21.955760372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:24:21.962960775Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T00:24:22.30982213Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:24:22.309837332Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:24:22.309857854Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:24:22.309866158Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:24:22.733612626Z caller=checkpoint.go:378 msg="writing series" size="199 kB"
level=debug ts=2026-04-21T00:24:25.687710833Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:24:26.536271163Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=warn ts=2026-04-21T00:24:27.310122196Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:24:29.170613904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T00:24:30.338757201Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=warn ts=2026-04-21T00:24:32.310299431Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:24:34.141940881Z caller=checkpoint.go:378 msg="writing series" size="131 kB"
ts=2026-04-21T00:24:37.10050352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:24:37.310362789Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:24:37.968317034Z caller=checkpoint.go:378 msg="writing series" size="23 MB"
level=debug ts=2026-04-21T00:24:37.968346843Z caller=checkpoint.go:390 msg="flushing series" totalSize="23 MB" series=5
level=info ts=2026-04-21T00:24:40.069035702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:24:40.077146284Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:24:41.747727989Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=warn ts=2026-04-21T00:24:42.310157985Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:24:45.55183906Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-21T00:24:45.551865217Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=2
level=warn ts=2026-04-21T00:24:47.310125924Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:24:49.35308788Z caller=checkpoint.go:378 msg="writing series" size="270 kB"
level=info ts=2026-04-21T00:24:52.286786391Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:24:52.28781742Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:24:52.310178142Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:24:52.3102106Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-21T00:24:52.310203728Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:24:52.310221456Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:24:53.155665777Z caller=checkpoint.go:378 msg="writing series" size="7.2 kB"
level=info ts=2026-04-21T00:24:54.647938885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="122 B" avg_comp="122 B" total_uncomp="47 kB" avg_uncomp="47 kB" idle=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T00:24:54.648071258Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-04-21T00:24:54.653935726Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T00:24:54.948010659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:24:54.952045587Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T00:24:55.248932679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="325 B" avg_comp="325 B" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:24:55.253237169Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T00:24:55.54896704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="142 kB" avg_uncomp="142 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:24:55.552888285Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T00:24:55.848695474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="344 B" avg_uncomp="344 B" idle=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:24:55.85259244Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T00:24:56.148550872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="141 B" avg_comp="141 B" total_uncomp="30 kB" avg_uncomp="30 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:24:56.152409161Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T00:24:56.44835091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="260 B" avg_comp="260 B" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:24:56.45202386Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T00:24:56.748040024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:24:56.751704389Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T00:24:56.958488531Z caller=checkpoint.go:378 msg="writing series" size="77 kB"
level=warn ts=2026-04-21T00:24:57.310387863Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:24:57.389681299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:24:57.394153633Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-21T00:24:58.825420679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:53774"
level=debug ts=2026-04-21T00:25:00.761520771Z caller=checkpoint.go:378 msg="writing series" size="287 kB"
level=warn ts=2026-04-21T00:25:02.309570396Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:25:04.564130652Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-04-21T00:25:05.131267251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:25:05.135784135Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T00:25:05.485359532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:25:05.492329168Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T00:25:06.116041373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:25:06.119566087Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T00:25:06.813159876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:25:06.816561754Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-04-21T00:25:07.102200302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:25:07.310527694Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:25:08.367478947Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=info ts=2026-04-21T00:25:10.957540652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:25:10.961135574Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T00:25:11.341366497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:25:11.345272303Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T00:25:11.829824441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:25:11.833837509Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T00:25:12.169915658Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=warn ts=2026-04-21T00:25:12.309495504Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:25:12.590485872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:25:12.593999038Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T00:25:13.98067194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:25:13.984269213Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T00:25:15.097668405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:25:15.1045711Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:25:15.972707111Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-21T00:25:17.310426498Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:25:19.775977558Z caller=checkpoint.go:378 msg="writing series" size="200 kB"
level=info ts=2026-04-21T00:25:20.586270231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:25:20.593468448Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:25:20.654130752Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T00:25:22.309830817Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:25:22.309856166Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:25:22.309880014Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:25:22.30989178Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:25:23.445084164Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:25:23.445142145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="192 B" avg_comp="192 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:25:23.448746056Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T00:25:23.578347911Z caller=checkpoint.go:378 msg="writing series" size="246 kB"
level=debug ts=2026-04-21T00:25:23.578372677Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-04-21T00:25:27.31041761Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:25:27.381768631Z caller=checkpoint.go:378 msg="writing series" size="269 kB"
level=info ts=2026-04-21T00:25:31.122317516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:25:31.126500122Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T00:25:31.183790776Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=warn ts=2026-04-21T00:25:32.310548054Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:25:34.987014691Z caller=checkpoint.go:378 msg="writing series" size="6.1 kB"
level=info ts=2026-04-21T00:25:35.405665939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:25:35.409700484Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T00:25:35.736614489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:25:35.740326245Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T00:25:36.488595111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:25:36.492197671Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-04-21T00:25:37.104615502Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=info ts=2026-04-21T00:25:37.262800999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:25:37.266363654Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-21T00:25:37.310398765Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:25:37.535359996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:25:37.538847846Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T00:25:38.790036716Z caller=checkpoint.go:378 msg="writing series" size="305 kB"
level=info ts=2026-04-21T00:25:39.895325283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:25:39.899490289Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:25:40.099225157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:25:40.102972663Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-21T00:25:42.310002443Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:25:42.592580208Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=info ts=2026-04-21T00:25:44.625139348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:25:44.630728524Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T00:25:46.395522449Z caller=checkpoint.go:378 msg="writing series" size="192 kB"
level=info ts=2026-04-21T00:25:46.828742788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:25:46.835511691Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T00:25:47.309753578Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:25:47.453108138Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:25:47.471157738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:25:47.474491086Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T00:25:50.198539325Z caller=checkpoint.go:378 msg="writing series" size="104 kB"
level=info ts=2026-04-21T00:25:52.286846436Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:25:52.287255995Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:25:52.309508652Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:25:52.309546304Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:25:52.309557196Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:25:52.309553321Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:25:54.001370869Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-04-21T00:25:54.745747523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:25:54.749850119Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T00:25:56.681291878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:25:56.688315887Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:25:56.688334633Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:25:56.688362985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:25:56.69493132Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-21T00:25:57.310275331Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:25:57.804256255Z caller=checkpoint.go:378 msg="writing series" size="387 kB"
level=debug ts=2026-04-21T00:25:57.804281774Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=9
ts=2026-04-21T00:25:58.828940727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:42314"
level=debug ts=2026-04-21T00:26:01.612068498Z caller=checkpoint.go:378 msg="writing series" size="10 MB"
level=debug ts=2026-04-21T00:26:01.612094806Z caller=checkpoint.go:390 msg="flushing series" totalSize="10 MB" series=1
level=warn ts=2026-04-21T00:26:02.309511503Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:26:05.409868978Z caller=checkpoint.go:378 msg="writing series" size="657 B"
ts=2026-04-21T00:26:07.106307795Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:26:07.310254981Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:26:08.61104513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:26:08.616925979Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T00:26:08.828598062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:26:08.835990201Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:26:09.212359298Z caller=checkpoint.go:378 msg="writing series" size="501 kB"
level=warn ts=2026-04-21T00:26:12.310434404Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:26:13.015213094Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=debug ts=2026-04-21T00:26:15.661047498Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:26:16.818483083Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=info ts=2026-04-21T00:26:16.864637765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:26:16.871058131Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T00:26:17.310078677Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:26:18.283303196Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T00:26:19.66637514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:26:19.669885161Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T00:26:20.620369438Z caller=checkpoint.go:378 msg="writing series" size="143 kB"
level=info ts=2026-04-21T00:26:22.309538491Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:26:22.309570851Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:26:22.309579686Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:26:22.309647791Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:26:24.424037429Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=info ts=2026-04-21T00:26:25.224131214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:26:25.228217346Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T00:26:27.008450042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:26:27.012041838Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-21T00:26:27.309676987Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:26:27.835394914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:26:27.839270854Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T00:26:28.2268345Z caller=checkpoint.go:378 msg="writing series" size="43 kB"
ts=2026-04-21T00:26:28.831304526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:38158"
level=info ts=2026-04-21T00:26:29.622384634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:26:29.626175028Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T00:26:30.470372363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:26:30.475007697Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:26:30.93338116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:26:30.939951847Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T00:26:30.9449822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:26:30.948249682Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T00:26:32.02872024Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-04-21T00:26:32.291863668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:26:32.295398682Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-21T00:26:32.309507656Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:26:32.85788022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:26:32.861450283Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T00:26:35.832325903Z caller=checkpoint.go:378 msg="writing series" size="817 B"
ts=2026-04-21T00:26:37.107999549Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=info ts=2026-04-21T00:26:37.149696232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:26:37.154066675Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T00:26:37.310425872Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:26:39.634863591Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=warn ts=2026-04-21T00:26:42.310257313Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:26:43.43822339Z caller=checkpoint.go:378 msg="writing series" size="841 kB"
level=debug ts=2026-04-21T00:26:43.438258703Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=11
level=info ts=2026-04-21T00:26:44.314790054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:26:44.322453283Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:26:46.507026888Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:26:47.241013134Z caller=checkpoint.go:378 msg="writing series" size="98 kB"
level=warn ts=2026-04-21T00:26:47.310132281Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:26:51.043386567Z caller=checkpoint.go:378 msg="writing series" size="553 kB"
level=info ts=2026-04-21T00:26:52.286897334Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:26:52.28737326Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:26:52.309681312Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:26:52.309711567Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-21T00:26:52.309704225Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:26:52.309720314Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T00:26:53.920033073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:26:53.926068738Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T00:26:54.846751958Z caller=checkpoint.go:378 msg="writing series" size="295 kB"
ts=2026-04-21T00:26:54.9732581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:56360"
level=warn ts=2026-04-21T00:26:57.310396591Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:26:58.649487073Z caller=checkpoint.go:378 msg="writing series" size="80 kB"
ts=2026-04-21T00:26:58.83314171Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:49746"
level=warn ts=2026-04-21T00:27:02.310460091Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:27:02.451839102Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=debug ts=2026-04-21T00:27:02.451867147Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5
level=debug ts=2026-04-21T00:27:06.254419208Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
ts=2026-04-21T00:27:07.110026307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:27:07.310355793Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:27:07.823400956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:27:07.829383076Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T00:27:08.044118871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:27:08.048354291Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T00:27:10.057695361Z caller=checkpoint.go:378 msg="writing series" size="435 kB"
level=debug ts=2026-04-21T00:27:10.746093487Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T00:27:10.748021841Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T00:27:12.310457349Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:27:12.612953976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:27:12.620065259Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T00:27:13.860698621Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=info ts=2026-04-21T00:27:14.537767017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:27:14.541760663Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T00:27:15.234305618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:27:15.237951917Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T00:27:16.166748928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:27:16.170261383Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T00:27:16.67176671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:27:16.679016592Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:27:17.045243188Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:27:17.309767312Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:27:17.663131785Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-04-21T00:27:17.909220681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:27:17.912940909Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T00:27:19.666853784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:27:19.67058894Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T00:27:20.696824904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:27:20.700837885Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:27:21.465975483Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-04-21T00:27:22.310336714Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:27:22.310373121Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-21T00:27:22.310375437Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:27:22.310383472Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T00:27:24.242297253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:27:24.247695161Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T00:27:25.268397277Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=info ts=2026-04-21T00:27:26.080027224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:27:26.083463885Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-21T00:27:27.309751803Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:27:29.071849985Z caller=checkpoint.go:378 msg="writing series" size="244 kB"
level=warn ts=2026-04-21T00:27:32.310017925Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:27:32.874178094Z caller=checkpoint.go:378 msg="writing series" size="8.2 kB"
level=debug ts=2026-04-21T00:27:36.676666054Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=debug ts=2026-04-21T00:27:36.67669152Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
ts=2026-04-21T00:27:37.111905683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:27:37.31029462Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:27:39.361836939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:27:39.368544588Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T00:27:40.482552774Z caller=checkpoint.go:378 msg="writing series" size="2.4 MB"
level=debug ts=2026-04-21T00:27:40.482576386Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.4 MB" series=1
level=warn ts=2026-04-21T00:27:42.310417557Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:27:44.08874689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:27:44.097264137Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:27:44.283068474Z caller=checkpoint.go:378 msg="writing series" size="499 kB"
level=warn ts=2026-04-21T00:27:47.310297857Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:27:48.08609981Z caller=checkpoint.go:378 msg="writing series" size="869 kB"
level=debug ts=2026-04-21T00:27:48.086123811Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=2
level=debug ts=2026-04-21T00:27:50.934884543Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:27:51.888703618Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=info ts=2026-04-21T00:27:52.28665869Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:27:52.2866947Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:27:52.310114237Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:27:52.310142039Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:27:52.310151429Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:27:52.310135888Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:27:54.975988783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:40042"
level=info ts=2026-04-21T00:27:55.219293575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:27:55.224454399Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T00:27:55.691701022Z caller=checkpoint.go:378 msg="writing series" size="336 kB"
level=debug ts=2026-04-21T00:27:56.43691381Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T00:27:57.310363245Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:27:58.022355573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:27:58.029029499Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-21T00:27:58.838148903Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:44398"
level=debug ts=2026-04-21T00:27:59.494096949Z caller=checkpoint.go:378 msg="writing series" size="828 kB"
level=debug ts=2026-04-21T00:27:59.494120862Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=3
level=info ts=2026-04-21T00:28:02.177220791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:28:02.18108429Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-04-21T00:28:02.309546941Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:28:02.949988137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:28:02.953646553Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T00:28:03.297142927Z caller=checkpoint.go:378 msg="writing series" size="5.5 kB"
level=info ts=2026-04-21T00:28:07.079583727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:28:07.08326306Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T00:28:07.099504612Z caller=checkpoint.go:378 msg="writing series" size="437 kB"
ts=2026-04-21T00:28:07.114119831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:28:07.310356864Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:28:07.781843054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:28:07.785750331Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T00:28:10.902559206Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=info ts=2026-04-21T00:28:10.932678887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:28:10.936662848Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T00:28:11.402924873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:28:11.409907314Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:28:11.409934941Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:28:11.40996412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:28:11.416723086Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T00:28:12.292074332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:28:12.295546917Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-04-21T00:28:12.309648025Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:28:14.705699703Z caller=checkpoint.go:378 msg="writing series" size="191 kB"
level=debug ts=2026-04-21T00:28:15.648404207Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:28:17.309751707Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:28:18.50788592Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=info ts=2026-04-21T00:28:20.249385708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:28:20.253871912Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:28:21.938457365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:28:21.942530967Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T00:28:22.310347847Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:28:22.310357665Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:28:22.310364369Z caller=checkpoint.go:390 msg="flushing series" totalSize="672 kB" series=5
level=debug ts=2026-04-21T00:28:22.310382858Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:28:22.310393305Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T00:28:22.318583626Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000011.tmp new=/var/loki/wal/checkpoint.000011
level=info ts=2026-04-21T00:28:22.3224331Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.01210627s
level=info ts=2026-04-21T00:28:22.559327379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:28:22.566946652Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T00:28:24.024291112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:28:24.028182412Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T00:28:24.182674048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:28:24.187148908Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T00:28:27.310296859Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:28:28.840311229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:50654"
ts=2026-04-21T00:28:28.912702828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:44600"
level=warn ts=2026-04-21T00:28:32.3098711Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:28:37.115324956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:28:37.310178331Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:28:39.080066606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:28:39.090396117Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T00:28:41.083177514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:28:41.09196745Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-21T00:28:42.310522719Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:28:45.036598882Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T00:28:47.309761479Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:28:47.775651189Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:28:47.775655757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:28:47.781076689Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T00:28:48.282432342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:28:48.285868303Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T00:28:52.286432791Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:28:52.286436728Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:28:52.309819319Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-21T00:28:52.30980857Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:28:52.309847927Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:28:52.309858038Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:28:52.309862595Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:28:52.309946446Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000012
level=debug ts=2026-04-21T00:28:52.310153306Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=info ts=2026-04-21T00:28:52.874131991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:28:52.877768919Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T00:28:56.11356935Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-04-21T00:28:57.283727226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:28:57.287475782Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-21T00:28:57.309715757Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:28:58.689125154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:28:58.695573627Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-21T00:28:58.842849354Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:50358"
level=info ts=2026-04-21T00:28:59.315236933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:28:59.322564471Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T00:28:59.916393332Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=info ts=2026-04-21T00:29:00.485439955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:29:00.490662088Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T00:29:01.055242229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:29:01.05923887Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T00:29:01.941313089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:29:01.944854462Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-21T00:29:02.310260392Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:29:03.719751611Z caller=checkpoint.go:378 msg="writing series" size="251 kB"
level=info ts=2026-04-21T00:29:05.267020565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:29:05.27140853Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-21T00:29:07.116913384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:29:07.310001269Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:29:07.410267651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:29:07.413946522Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T00:29:07.521967286Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=info ts=2026-04-21T00:29:08.87319267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:29:08.877330492Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:29:08.877352758Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:29:08.877376698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:29:08.88101865Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:29:11.32527888Z caller=checkpoint.go:378 msg="writing series" size="688 kB"
level=info ts=2026-04-21T00:29:12.070241087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:29:12.076888876Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-21T00:29:12.309552743Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:29:14.553388709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:29:14.557399192Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T00:29:15.128363268Z caller=checkpoint.go:378 msg="writing series" size="966 kB"
level=debug ts=2026-04-21T00:29:15.128388274Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.0 MB" series=7
level=debug ts=2026-04-21T00:29:16.300531181Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:29:17.310095015Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:29:18.931367512Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-21T00:29:18.931390825Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-04-21T00:29:22.31011141Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:29:22.310109893Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:29:22.310141435Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:29:22.31015241Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:29:22.733524238Z caller=checkpoint.go:378 msg="writing series" size="205 kB"
level=info ts=2026-04-21T00:29:23.684496487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:29:23.688888638Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:29:24.168034747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:29:24.173526264Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-04-21T00:29:24.981246599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:42826"
level=debug ts=2026-04-21T00:29:26.53654649Z caller=checkpoint.go:378 msg="writing series" size="103 kB"
level=debug ts=2026-04-21T00:29:27.2105918Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T00:29:27.309683577Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:29:30.339172406Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=info ts=2026-04-21T00:29:31.290469671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:29:31.294051109Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-21T00:29:32.309551098Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:29:33.437249177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:29:33.445105796Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T00:29:33.622363877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:29:33.629410294Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:29:34.142132819Z caller=checkpoint.go:378 msg="writing series" size="140 kB"
ts=2026-04-21T00:29:37.118661603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:29:37.309840311Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:29:37.646856548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:29:37.650850321Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T00:29:37.968705595Z caller=checkpoint.go:378 msg="writing series" size="23 MB"
level=debug ts=2026-04-21T00:29:37.968731788Z caller=checkpoint.go:390 msg="flushing series" totalSize="24 MB" series=5
level=debug ts=2026-04-21T00:29:41.746855202Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=warn ts=2026-04-21T00:29:42.310382283Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:29:44.145456932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:29:44.151130319Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T00:29:45.551403439Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-21T00:29:45.551441142Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=2
level=warn ts=2026-04-21T00:29:47.31008777Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:29:48.894765637Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:29:49.353213002Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=info ts=2026-04-21T00:29:50.629764257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:29:50.637751908Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T00:29:52.077841991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:29:52.081540968Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T00:29:52.286384173Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:29:52.28674523Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:29:52.310169246Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:29:52.310196181Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:29:52.310206382Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:29:52.310164455Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:29:53.049395786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:29:53.052843714Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T00:29:53.156302584Z caller=checkpoint.go:378 msg="writing series" size="7.2 kB"
level=info ts=2026-04-21T00:29:55.582374702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:29:55.586594398Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:29:55.586617716Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:29:55.586659723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:29:55.59049875Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T00:29:55.930620546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:29:55.934345725Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T00:29:56.958279096Z caller=checkpoint.go:378 msg="writing series" size="77 kB"
level=warn ts=2026-04-21T00:29:57.310048363Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:29:58.917711966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:37114"
level=debug ts=2026-04-21T00:30:00.76159792Z caller=checkpoint.go:378 msg="writing series" size="280 kB"
level=info ts=2026-04-21T00:30:01.635135966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:30:01.639460413Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-21T00:30:02.309511259Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:30:02.682052994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:30:02.685562363Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T00:30:04.563966253Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=info ts=2026-04-21T00:30:06.307606333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:30:06.311475751Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T00:30:06.673382692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:30:06.677517804Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-04-21T00:30:07.120564372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:30:07.309538734Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:30:08.366960806Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=info ts=2026-04-21T00:30:11.261596356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:30:11.266290831Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T00:30:12.170086016Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=warn ts=2026-04-21T00:30:12.310525151Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:30:13.695295225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:30:13.703131959Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:30:14.391993947Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T00:30:15.973117174Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-21T00:30:17.310086867Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:30:17.331238469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:30:17.33497845Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T00:30:18.813694688Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:30:19.775561822Z caller=checkpoint.go:378 msg="writing series" size="213 kB"
level=info ts=2026-04-21T00:30:22.223443487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:30:22.229527854Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T00:30:22.309835372Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:30:22.309862878Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:30:22.309871625Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:30:22.30990307Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:30:23.466275645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:30:23.474176836Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:30:23.474199993Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:30:23.474227823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:30:23.480382787Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:30:23.57879276Z caller=checkpoint.go:378 msg="writing series" size="233 kB"
level=warn ts=2026-04-21T00:30:27.309718914Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:30:27.381142915Z caller=checkpoint.go:378 msg="writing series" size="269 kB"
level=debug ts=2026-04-21T00:30:27.381168245Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=11
level=info ts=2026-04-21T00:30:27.988106681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:30:27.992208937Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-04-21T00:30:28.849350973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:58896"
level=debug ts=2026-04-21T00:30:31.18365451Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=warn ts=2026-04-21T00:30:32.310479362Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:30:32.899195647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:30:32.905557254Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T00:30:33.665582805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:30:33.673729216Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:30:34.987051823Z caller=checkpoint.go:378 msg="writing series" size="6.7 kB"
ts=2026-04-21T00:30:37.122780474Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:30:37.310382208Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:30:38.789511782Z caller=checkpoint.go:378 msg="writing series" size="357 kB"
level=info ts=2026-04-21T00:30:38.922771164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:30:38.926606071Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-04-21T00:30:42.310415031Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:30:42.592946269Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=info ts=2026-04-21T00:30:43.124596934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:30:43.130033928Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T00:30:44.680770225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:30:44.685056141Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:30:46.394965618Z caller=checkpoint.go:378 msg="writing series" size="192 kB"
level=debug ts=2026-04-21T00:30:46.833187043Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:30:47.310234133Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:30:47.566670858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:30:47.570931603Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T00:30:48.59209434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:30:48.596023444Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T00:30:49.81767919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:30:49.821361111Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T00:30:50.198005478Z caller=checkpoint.go:378 msg="writing series" size="104 kB"
level=info ts=2026-04-21T00:30:52.286670696Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:30:52.287134789Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=debug ts=2026-04-21T00:30:52.287332822Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-21T00:30:52.287374106Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974145 n=1
level=debug ts=2026-04-21T00:30:52.287425499Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974145 n=1
level=info ts=2026-04-21T00:30:52.309459371Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:30:52.309483219Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:30:52.309492161Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:30:52.30950079Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:30:54.000859733Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-04-21T00:30:54.113308903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:30:54.117045065Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T00:30:55.522644422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:30:55.528016921Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-21T00:30:57.31037887Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:30:57.803959183Z caller=checkpoint.go:378 msg="writing series" size="494 kB"
level=debug ts=2026-04-21T00:30:57.803987656Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=8
ts=2026-04-21T00:30:58.92033476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:55208"
level=info ts=2026-04-21T00:31:00.046005017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:31:00.053570242Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T00:31:00.485347389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:31:00.489379884Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T00:31:01.613541241Z caller=checkpoint.go:378 msg="writing series" size="11 MB"
level=debug ts=2026-04-21T00:31:01.613566335Z caller=checkpoint.go:390 msg="flushing series" totalSize="11 MB" series=1
level=warn ts=2026-04-21T00:31:02.310223702Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:31:03.472746113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:31:03.476681318Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:31:03.476705784Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:31:03.476730746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:31:03.479749106Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T00:31:05.408820428Z caller=checkpoint.go:378 msg="writing series" size="657 B"
level=info ts=2026-04-21T00:31:05.520497172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:31:05.528837473Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-21T00:31:07.124360035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:31:07.310536069Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:31:08.287502458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:31:08.29224704Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-21T00:31:09.212109678Z caller=checkpoint.go:378 msg="writing series" size="506 kB"
level=debug ts=2026-04-21T00:31:11.548173183Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T00:31:11.549821385Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T00:31:12.310036767Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:31:13.015201236Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=info ts=2026-04-21T00:31:14.2763399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:31:14.279867257Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T00:31:16.818213686Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=warn ts=2026-04-21T00:31:17.310421888Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:31:18.186819428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:31:18.191995879Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:31:18.1920199Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:31:18.192046101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:31:18.1949971Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T00:31:20.620293409Z caller=checkpoint.go:378 msg="writing series" size="150 kB"
level=debug ts=2026-04-21T00:31:22.160770604Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:31:22.310179463Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:31:22.310211544Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:31:22.310220543Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:31:22.310240927Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:31:24.423157042Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
ts=2026-04-21T00:31:24.987973517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:45992"
level=info ts=2026-04-21T00:31:25.901611073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:31:25.905952818Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:31:25.905973293Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:31:25.905996959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:31:25.909455284Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-04-21T00:31:27.310540979Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:31:28.225955023Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
ts=2026-04-21T00:31:28.853472502Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:58806"
level=info ts=2026-04-21T00:31:31.489937103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:31:31.496966497Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T00:31:31.996691187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:31:32.000857151Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:31:32.02894973Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=warn ts=2026-04-21T00:31:32.309718044Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:31:35.832196743Z caller=checkpoint.go:378 msg="writing series" size="817 B"
level=info ts=2026-04-21T00:31:36.338889722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:31:36.34257011Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-04-21T00:31:37.126173125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:31:37.310335764Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:31:37.703378997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:31:37.70715701Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:31:37.707176117Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:31:37.707196825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:31:37.710311968Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T00:31:37.790580806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:31:37.793978786Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T00:31:39.634653801Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=info ts=2026-04-21T00:31:40.832161059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:31:40.835740801Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:31:40.835763119Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:31:40.835785734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:31:40.838752853Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T00:31:42.071311019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:31:42.078254626Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-21T00:31:42.309840583Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:31:42.700719519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:31:42.704678752Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T00:31:43.437882386Z caller=checkpoint.go:378 msg="writing series" size="841 kB"
level=debug ts=2026-04-21T00:31:43.437912295Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=11
level=debug ts=2026-04-21T00:31:45.77819445Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T00:31:46.094177155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:31:46.098512791Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T00:31:47.240477679Z caller=checkpoint.go:378 msg="writing series" size="122 kB"
level=warn ts=2026-04-21T00:31:47.310260312Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:31:48.316284115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:31:48.320774336Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:31:48.320798036Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:31:48.320821987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:31:48.323985483Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T00:31:48.636465851Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:31:49.000579294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:31:49.004644196Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:31:49.004667265Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:31:49.004692159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:31:49.007760248Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T00:31:50.593819989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:31:50.597379404Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T00:31:51.043816954Z caller=checkpoint.go:378 msg="writing series" size="416 kB"
level=info ts=2026-04-21T00:31:52.286533955Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:31:52.286538753Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:31:52.309882861Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:31:52.309911389Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-21T00:31:52.309901782Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:31:52.30992018Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T00:31:52.609502569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:31:52.612996737Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:31:52.61301683Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:31:52.613038663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:31:52.615954048Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T00:31:53.082008048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:31:53.085592639Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-04-21T00:31:53.085613461Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:31:53.08564954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:31:53.088733642Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T00:31:54.606093742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:31:54.611348859Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-21T00:31:54.846195569Z caller=checkpoint.go:378 msg="writing series" size="321 kB"
level=info ts=2026-04-21T00:31:57.189005958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:31:57.194055073Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-21T00:31:57.31022701Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:31:57.711636681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:31:57.719179102Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:31:58.648593513Z caller=checkpoint.go:378 msg="writing series" size="82 kB"
level=warn ts=2026-04-21T00:32:02.309696096Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:32:02.45107759Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=info ts=2026-04-21T00:32:03.376243279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:32:03.379894119Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T00:32:04.000218353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:32:04.003772683Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T00:32:06.254348346Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
ts=2026-04-21T00:32:07.127834822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=info ts=2026-04-21T00:32:07.233924106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:32:07.237807984Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-21T00:32:07.310386176Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:32:07.975555083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:32:07.979148416Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T00:32:08.479533704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:32:08.483760838Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T00:32:10.057343569Z caller=checkpoint.go:378 msg="writing series" size="435 kB"
level=debug ts=2026-04-21T00:32:10.057369195Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=7
level=info ts=2026-04-21T00:32:11.866613711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:32:11.870453976Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-04-21T00:32:12.309769905Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:32:13.860097628Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=warn ts=2026-04-21T00:32:17.310474307Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:32:17.662704907Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-04-21T00:32:20.461945845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:32:20.467830406Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T00:32:21.465588496Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=debug ts=2026-04-21T00:32:21.506101864Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:32:21.670339731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:32:21.674644238Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T00:32:22.18933115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:32:22.193552257Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:32:22.310225784Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:32:22.310263042Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:32:22.310273154Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:32:22.310269165Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:32:23.867865122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:32:23.874387629Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-21T00:32:24.991538998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:54840"
level=debug ts=2026-04-21T00:32:25.108021304Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T00:32:25.268755409Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=info ts=2026-04-21T00:32:26.854179656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:32:26.861615595Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:32:26.861656688Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:32:26.861689865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:32:26.868164558Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-21T00:32:27.309826588Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:32:28.926217609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:53052"
level=debug ts=2026-04-21T00:32:29.071071338Z caller=checkpoint.go:378 msg="writing series" size="285 kB"
level=info ts=2026-04-21T00:32:31.048143936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:32:31.052079493Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-21T00:32:32.310437958Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:32:32.874135309Z caller=checkpoint.go:378 msg="writing series" size="8.2 kB"
level=info ts=2026-04-21T00:32:32.906508986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:32:32.910720737Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T00:32:35.999836177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:32:36.005322563Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-21T00:32:36.677182023Z caller=checkpoint.go:378 msg="writing series" size="104 kB"
ts=2026-04-21T00:32:37.129718854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:32:37.310407556Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:32:37.92739303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:32:37.931546668Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T00:32:40.481014594Z caller=checkpoint.go:378 msg="writing series" size="2.5 MB"
level=debug ts=2026-04-21T00:32:40.481040157Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.1 MB" series=8
level=info ts=2026-04-21T00:32:40.617899347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:32:40.621805641Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-04-21T00:32:42.309835047Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:32:44.282293012Z caller=checkpoint.go:378 msg="writing series" size="416 kB"
level=info ts=2026-04-21T00:32:44.658129245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:32:44.662197144Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T00:32:46.258951696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:32:46.266071262Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T00:32:46.53306059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:32:46.536605756Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-21T00:32:47.310467189Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:32:48.085675754Z caller=checkpoint.go:378 msg="writing series" size="869 kB"
level=debug ts=2026-04-21T00:32:48.085707422Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=2
level=debug ts=2026-04-21T00:32:48.92553484Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:32:51.853680404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:32:51.861714853Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:32:51.888110457Z caller=checkpoint.go:378 msg="writing series" size="4.5 kB"
level=info ts=2026-04-21T00:32:52.286452953Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:32:52.286909994Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:32:52.310354406Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:32:52.310380639Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-21T00:32:52.310373404Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:32:52.310389534Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:32:55.691119501Z caller=checkpoint.go:378 msg="writing series" size="336 kB"
level=info ts=2026-04-21T00:32:55.735978213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:32:55.73952448Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-21T00:32:57.309898242Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:32:57.779832793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:32:57.783559487Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-04-21T00:32:58.928025873Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:55340"
level=debug ts=2026-04-21T00:32:59.494427879Z caller=checkpoint.go:378 msg="writing series" size="753 kB"
level=debug ts=2026-04-21T00:32:59.494457613Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=3
level=warn ts=2026-04-21T00:33:02.309525967Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:33:02.463144549Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T00:33:03.296226278Z caller=checkpoint.go:378 msg="writing series" size="5.5 kB"
level=info ts=2026-04-21T00:33:03.447555691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:33:03.452045427Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:33:07.099247098Z caller=checkpoint.go:378 msg="writing series" size="437 kB"
ts=2026-04-21T00:33:07.131470463Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:33:07.309859214Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:33:10.902238131Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=info ts=2026-04-21T00:33:12.051082658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:33:12.05770375Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T00:33:12.310413062Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:33:14.70471794Z caller=checkpoint.go:378 msg="writing series" size="192 kB"
level=info ts=2026-04-21T00:33:16.233015681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:33:16.237505754Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T00:33:16.324015862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:33:16.329207917Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-21T00:33:16.998151272Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:33:17.309828783Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:33:18.508251976Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=info ts=2026-04-21T00:33:19.725992097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:33:19.729832697Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T00:33:20.677986526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:33:20.682047902Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T00:33:22.310026384Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:33:22.310036731Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:33:22.310055285Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:33:22.310063639Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:33:22.311124105Z caller=checkpoint.go:390 msg="flushing series" totalSize="673 kB" series=5
level=info ts=2026-04-21T00:33:22.318573079Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000012.tmp new=/var/loki/wal/checkpoint.000012
level=info ts=2026-04-21T00:33:22.322827495Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.012712561s
level=info ts=2026-04-21T00:33:23.456861716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:33:23.462101895Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T00:33:25.66836905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:33:25.675653329Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-21T00:33:27.310517448Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:33:28.930133843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:44842"
level=info ts=2026-04-21T00:33:30.386487154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:33:30.39022466Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-21T00:33:32.309582561Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:33:35.36637794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:33:35.371011013Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-04-21T00:33:37.132451899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:33:37.310146807Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:33:38.181813336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:33:38.185949437Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T00:33:38.600085177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:33:38.604151758Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T00:33:40.535948604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:33:40.540034551Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-21T00:33:42.309974894Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:33:42.469334223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:33:42.476093958Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T00:33:47.309841188Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:33:48.234943012Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:33:48.235008983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:33:48.239692421Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-21T00:33:52.286375609Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:33:52.286419104Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:33:52.309727941Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-04-21T00:33:52.309752794Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:33:52.309761902Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:33:52.30978398Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:33:52.309795193Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T00:33:52.310020915Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000013
level=debug ts=2026-04-21T00:33:52.310216427Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=info ts=2026-04-21T00:33:54.063274344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:33:54.06707294Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T00:33:54.5699443Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T00:33:54.996360113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:45876"
level=debug ts=2026-04-21T00:33:56.114060271Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=warn ts=2026-04-21T00:33:57.310371007Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:33:57.389680358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:33:57.396008539Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-21T00:33:58.931676176Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:38460"
level=info ts=2026-04-21T00:33:59.273037919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:33:59.276838409Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T00:33:59.916181345Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-21T00:34:02.309838637Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:34:02.459214193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:34:02.463238723Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-21T00:34:02.93437287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:34:02.93847014Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:34:03.719823031Z caller=checkpoint.go:378 msg="writing series" size="301 kB"
ts=2026-04-21T00:34:07.134341905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:34:07.309681079Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:34:07.521891298Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
level=info ts=2026-04-21T00:34:09.483410523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:34:09.488465375Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T00:34:09.503528977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:34:09.510155369Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:34:11.32561543Z caller=checkpoint.go:378 msg="writing series" size="688 kB"
level=debug ts=2026-04-21T00:34:11.325662465Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-04-21T00:34:12.310071015Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:34:15.128716319Z caller=checkpoint.go:378 msg="writing series" size="967 kB"
level=info ts=2026-04-21T00:34:15.638691398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:34:15.644241987Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-21T00:34:17.3102536Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:34:18.931314183Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-04-21T00:34:18.931342208Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.2 MB" series=2
level=info ts=2026-04-21T00:34:19.543321235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:34:19.547352786Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T00:34:20.590046213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:34:20.593754787Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T00:34:21.504285905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:34:21.508459545Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T00:34:22.309435601Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:34:22.309469059Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:34:22.309478076Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:34:22.309501247Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:34:22.349675189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:34:22.353034127Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T00:34:22.733646786Z caller=checkpoint.go:378 msg="writing series" size="208 kB"
level=debug ts=2026-04-21T00:34:24.023727596Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:34:24.95040142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:34:24.956803548Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T00:34:26.535977701Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=info ts=2026-04-21T00:34:27.084610836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:34:27.089375524Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-21T00:34:27.310589129Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:34:28.196280675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:34:28.200347532Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-21T00:34:28.863715243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:49476"
level=debug ts=2026-04-21T00:34:30.339468254Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=warn ts=2026-04-21T00:34:32.310446403Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:34:34.141947182Z caller=checkpoint.go:378 msg="writing series" size="148 kB"
level=info ts=2026-04-21T00:34:35.859220325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:34:35.863821086Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-21T00:34:37.054290086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:34:37.058395594Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-04-21T00:34:37.135945656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:34:37.309658727Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:34:37.964833814Z caller=checkpoint.go:378 msg="writing series" size="23 MB"
level=debug ts=2026-04-21T00:34:37.96486056Z caller=checkpoint.go:390 msg="flushing series" totalSize="24 MB" series=5
level=info ts=2026-04-21T00:34:41.438923133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:34:41.445739239Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:34:41.445762302Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:34:41.445789056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:34:41.452149435Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T00:34:41.747525032Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=info ts=2026-04-21T00:34:42.003333857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:34:42.009800074Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T00:34:42.309516286Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:34:45.55044493Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB"
level=debug ts=2026-04-21T00:34:45.550477514Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=2
level=warn ts=2026-04-21T00:34:47.310331731Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:34:49.353257466Z caller=checkpoint.go:378 msg="writing series" size="106 kB"
level=debug ts=2026-04-21T00:34:51.332102635Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T00:34:51.333870854Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T00:34:52.286403032Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:34:52.286819181Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:34:52.310239596Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:34:52.310265143Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-21T00:34:52.310254008Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:34:52.310273163Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:34:53.155513924Z caller=checkpoint.go:378 msg="writing series" size="7.2 kB"
level=info ts=2026-04-21T00:34:54.157084178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:34:54.166036446Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T00:34:54.331855955Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:34:55.62959591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:34:55.634571077Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:34:56.958459589Z caller=checkpoint.go:378 msg="writing series" size="78 kB"
level=warn ts=2026-04-21T00:34:57.310241612Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:34:58.57718773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:34:58.580807154Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-04-21T00:34:58.865517839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:37718"
ts=2026-04-21T00:34:58.935817197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:58858"
level=info ts=2026-04-21T00:34:59.952156764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:34:59.957119174Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:34:59.957142746Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:34:59.957168704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:34:59.961559461Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-21T00:35:00.762081384Z caller=checkpoint.go:378 msg="writing series" size="280 kB"
level=warn ts=2026-04-21T00:35:02.3096795Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:35:03.207193173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:35:03.211110194Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T00:35:03.914262409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:35:03.917768786Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T00:35:04.564590429Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=info ts=2026-04-21T00:35:06.258617102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:35:06.263229664Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T00:35:06.489970564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:35:06.496780612Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-21T00:35:07.13810369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:35:07.310218548Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:35:08.36732558Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=info ts=2026-04-21T00:35:10.226841753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:35:10.230914899Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T00:35:12.170202437Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=warn ts=2026-04-21T00:35:12.310046924Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:35:13.057122408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:35:13.061402035Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T00:35:15.228616139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:35:15.232477339Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T00:35:15.973053621Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-21T00:35:17.102520603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:35:17.107830267Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-21T00:35:17.309998851Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:35:18.394440022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:35:18.398480922Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T00:35:18.777672484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:35:18.783845892Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T00:35:19.775888227Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=debug ts=2026-04-21T00:35:20.881377935Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:35:22.31027919Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:35:22.310287211Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:35:22.310308039Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:35:22.310317923Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:35:23.578583956Z caller=checkpoint.go:378 msg="writing series" size="260 kB"
ts=2026-04-21T00:35:25.001019314Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:47674"
level=info ts=2026-04-21T00:35:25.02658811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:35:25.031301342Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T00:35:25.502262955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:35:25.50906346Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T00:35:27.310070422Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:35:27.381497232Z caller=checkpoint.go:378 msg="writing series" size="269 kB"
level=debug ts=2026-04-21T00:35:27.381520069Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=11
level=info ts=2026-04-21T00:35:27.498465853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:35:27.503931144Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-04-21T00:35:28.868090026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:42072"
level=debug ts=2026-04-21T00:35:31.184055593Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=info ts=2026-04-21T00:35:32.286932832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:35:32.290817543Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-04-21T00:35:32.309969885Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:35:32.643047961Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T00:35:34.98685876Z caller=checkpoint.go:378 msg="writing series" size="6.7 kB"
ts=2026-04-21T00:35:37.139306661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:35:37.310152477Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:35:38.790165926Z caller=checkpoint.go:378 msg="writing series" size="223 kB"
level=info ts=2026-04-21T00:35:40.255166137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:35:40.259326553Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:35:40.259351511Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:35:40.259386537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:35:40.262749553Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-21T00:35:42.309705792Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:35:42.592239588Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=info ts=2026-04-21T00:35:45.939140417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:35:45.943555156Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T00:35:46.395090344Z caller=checkpoint.go:378 msg="writing series" size="192 kB"
level=warn ts=2026-04-21T00:35:47.309819941Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:35:47.62167626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:35:47.625424613Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T00:35:49.739928007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:35:49.74379689Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T00:35:50.198350289Z caller=checkpoint.go:378 msg="writing series" size="104 kB"
level=debug ts=2026-04-21T00:35:51.852051883Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:35:51.862103716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:35:51.867224344Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-21T00:35:52.28625992Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:35:52.287334909Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-21T00:35:52.309671744Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:35:52.309676608Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:35:52.30970184Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:35:52.30971399Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:35:54.000959492Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-04-21T00:35:57.098152101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:35:57.102559457Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:35:57.224832144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:35:57.231851797Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-21T00:35:57.310005331Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:35:57.804296318Z caller=checkpoint.go:378 msg="writing series" size="406 kB"
level=debug ts=2026-04-21T00:35:57.804321234Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-04-21T00:35:59.179794999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:35:59.18429228Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T00:35:59.487302353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:35:59.491335523Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T00:36:01.354771212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:36:01.358738266Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T00:36:01.61269516Z caller=checkpoint.go:378 msg="writing series" size="12 MB"
level=debug ts=2026-04-21T00:36:01.612722307Z caller=checkpoint.go:390 msg="flushing series" totalSize="12 MB" series=1
level=warn ts=2026-04-21T00:36:02.31036668Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:36:02.628963526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:36:02.635902188Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T00:36:05.409899361Z caller=checkpoint.go:378 msg="writing series" size="657 B"
level=info ts=2026-04-21T00:36:06.635874408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:36:06.639718075Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-04-21T00:36:07.141012942Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:36:07.309884545Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:36:09.21244917Z caller=checkpoint.go:378 msg="writing series" size="517 kB"
level=info ts=2026-04-21T00:36:11.601884966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:36:11.606482931Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T00:36:12.309680708Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:36:13.015269966Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=debug ts=2026-04-21T00:36:14.98973196Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T00:36:16.069366168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:36:16.072949125Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T00:36:16.81826726Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=warn ts=2026-04-21T00:36:17.310287942Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:36:17.374471909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:36:17.381554795Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T00:36:17.763430058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:36:17.767256492Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T00:36:18.127751957Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:36:19.070697754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:36:19.074815015Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-21T00:36:20.620326703Z caller=checkpoint.go:378 msg="writing series" size="156 kB"
level=info ts=2026-04-21T00:36:22.309432947Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:36:22.309480615Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:36:22.30949025Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:36:22.309508289Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:36:24.422973094Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=info ts=2026-04-21T00:36:26.171659574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:36:26.175431542Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-21T00:36:27.3095093Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:36:27.483940681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:36:27.488501127Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:36:28.226343092Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
ts=2026-04-21T00:36:28.871207315Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:53930"
level=debug ts=2026-04-21T00:36:32.029217309Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=warn ts=2026-04-21T00:36:32.310259996Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:36:33.868681573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:36:33.877774169Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T00:36:35.832227189Z caller=checkpoint.go:378 msg="writing series" size="817 B"
level=info ts=2026-04-21T00:36:36.439937721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:36:36.443769917Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-04-21T00:36:37.142896712Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:36:37.310418908Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:36:39.63518229Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=info ts=2026-04-21T00:36:39.980650536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:36:39.988018308Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-21T00:36:42.310176939Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:36:42.377440114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:36:42.381172298Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T00:36:43.438059532Z caller=checkpoint.go:378 msg="writing series" size="841 kB"
level=debug ts=2026-04-21T00:36:43.438084754Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=11
level=info ts=2026-04-21T00:36:45.224355483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:36:45.231074309Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T00:36:47.240232195Z caller=checkpoint.go:378 msg="writing series" size="147 kB"
level=warn ts=2026-04-21T00:36:47.310384232Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:36:48.581569816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:36:48.587156323Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T00:36:49.655245295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:36:49.658967583Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T00:36:51.043113128Z caller=checkpoint.go:378 msg="writing series" size="422 kB"
level=info ts=2026-04-21T00:36:52.286380241Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:36:52.28660593Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:36:52.310041715Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:36:52.310069846Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:36:52.310080961Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:36:52.310039076Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:36:53.251865492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:36:53.256771904Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:36:53.256806518Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:36:53.256847761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:36:53.260950631Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T00:36:53.521167672Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:36:54.27490444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:36:54.27838734Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T00:36:54.846607915Z caller=checkpoint.go:378 msg="writing series" size="321 kB"
level=warn ts=2026-04-21T00:36:57.30985503Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:36:57.319991123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:36:57.323885392Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T00:36:58.649279822Z caller=checkpoint.go:378 msg="writing series" size="88 kB"
level=info ts=2026-04-21T00:36:59.253077609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:36:59.25990664Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:36:59.259932159Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:36:59.259966014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:36:59.266240086Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T00:36:59.464036246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:36:59.468218092Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-04-21T00:37:02.309907102Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:37:02.401193278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:37:02.405064976Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T00:37:02.45115245Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=info ts=2026-04-21T00:37:02.564198566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:37:02.567649273Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T00:37:04.033863716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:37:04.040114337Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T00:37:06.254533361Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
ts=2026-04-21T00:37:07.144901637Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:37:07.310199803Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:37:10.057208061Z caller=checkpoint.go:378 msg="writing series" size="435 kB"
level=debug ts=2026-04-21T00:37:10.057232195Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=7
level=debug ts=2026-04-21T00:37:11.432429938Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T00:37:12.310023805Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:37:12.673673414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:37:12.680587582Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T00:37:13.819350781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:37:13.822828272Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T00:37:13.860076828Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=info ts=2026-04-21T00:37:15.673053059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:37:15.677131354Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-21T00:37:17.310364388Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:37:17.662968216Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-04-21T00:37:21.363349795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:37:21.367068209Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T00:37:21.465490501Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-04-21T00:37:22.281093306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:37:22.288558468Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-21T00:37:22.309690862Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:37:22.309714562Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:37:22.309749361Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:37:22.309758108Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-21T00:37:25.007231602Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:59702"
level=debug ts=2026-04-21T00:37:25.26834848Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=warn ts=2026-04-21T00:37:27.310420641Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:37:28.875005566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:60892"
ts=2026-04-21T00:37:28.944498108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:35576"
level=debug ts=2026-04-21T00:37:29.071456574Z caller=checkpoint.go:378 msg="writing series" size="341 kB"
level=debug ts=2026-04-21T00:37:29.162864027Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:37:30.221013081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:37:30.227348794Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T00:37:32.309662057Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:37:32.606240492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:37:32.609969185Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T00:37:32.874136333Z caller=checkpoint.go:378 msg="writing series" size="8.2 kB"
level=debug ts=2026-04-21T00:37:36.677378722Z caller=checkpoint.go:378 msg="writing series" size="107 kB"
ts=2026-04-21T00:37:37.147361413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:37:37.309524494Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:37:38.006546174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:37:38.012942379Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T00:37:38.035002786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:37:38.042998651Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T00:37:39.779417413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:37:39.783317241Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T00:37:40.481450307Z caller=checkpoint.go:378 msg="writing series" size="2.5 MB"
level=debug ts=2026-04-21T00:37:40.481474521Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.3 MB" series=8
level=info ts=2026-04-21T00:37:41.071642164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:37:41.078279953Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T00:37:41.80847111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:37:41.812464677Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T00:37:41.911853634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:37:41.915478248Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:37:41.91550252Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:37:41.915524997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:37:41.918563227Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-21T00:37:42.309728381Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:37:43.220256945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:37:43.224889251Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-21T00:37:44.28234122Z caller=checkpoint.go:378 msg="writing series" size="502 kB"
level=info ts=2026-04-21T00:37:46.314100154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:37:46.317778524Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:37:46.317803149Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:37:46.31782763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:37:46.320698026Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-21T00:37:47.310062986Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:37:48.085813497Z caller=checkpoint.go:378 msg="writing series" size="869 kB"
level=debug ts=2026-04-21T00:37:48.085839902Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=2
level=debug ts=2026-04-21T00:37:51.887755342Z caller=checkpoint.go:378 msg="writing series" size="4.7 kB"
level=info ts=2026-04-21T00:37:52.286411898Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:37:52.286659531Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:37:52.310103897Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:37:52.310135973Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:37:52.310146116Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:37:52.310144974Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:37:55.121655127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:37:55.125394171Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T00:37:55.69168029Z caller=checkpoint.go:378 msg="writing series" size="336 kB"
level=debug ts=2026-04-21T00:37:55.838931754Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:37:57.309827863Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:37:58.489620254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:37:58.493743926Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:37:58.493765635Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:37:58.493790125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:37:58.496743193Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-04-21T00:37:58.877392683Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:60210"
level=info ts=2026-04-21T00:37:58.920125004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:37:58.924118699Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:37:58.924139584Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:37:58.924162192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:37:58.92721981Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T00:37:59.494491705Z caller=checkpoint.go:378 msg="writing series" size="834 kB"
level=debug ts=2026-04-21T00:37:59.494515909Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=3
level=info ts=2026-04-21T00:38:01.180669682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:38:01.184875469Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:38:01.284262497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:38:01.287720033Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T00:38:01.923500551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:38:01.927177036Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-21T00:38:02.310218511Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:38:03.296958935Z caller=checkpoint.go:378 msg="writing series" size="5.5 kB"
level=info ts=2026-04-21T00:38:03.865220696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:38:03.869141556Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:38:03.869171222Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:38:03.869201647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:38:03.8725744Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T00:38:04.967821753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:38:04.975286555Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:38:07.09953433Z caller=checkpoint.go:378 msg="writing series" size="437 kB"
ts=2026-04-21T00:38:07.149236381Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:38:07.310483968Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:38:09.830151473Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T00:38:10.902514791Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=info ts=2026-04-21T00:38:12.228225421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:38:12.231917953Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-21T00:38:12.31007008Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:38:14.705513293Z caller=checkpoint.go:378 msg="writing series" size="193 kB"
level=info ts=2026-04-21T00:38:15.241100575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:38:15.244797351Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-04-21T00:38:17.310000944Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:38:17.829755617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:38:17.833183162Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:38:17.833201638Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:38:17.833222709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:38:17.836092245Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T00:38:18.507659798Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=info ts=2026-04-21T00:38:19.076215767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:38:19.079705955Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-21T00:38:22.30983547Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:38:22.30984386Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:38:22.309868626Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:38:22.309878639Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:38:22.310920188Z caller=checkpoint.go:390 msg="flushing series" totalSize="673 kB" series=5
level=info ts=2026-04-21T00:38:22.316354384Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000013.tmp new=/var/loki/wal/checkpoint.000013
level=info ts=2026-04-21T00:38:22.319573416Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.009394855s
level=info ts=2026-04-21T00:38:24.324674612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:38:24.328611089Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T00:38:24.358691678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:38:24.362131919Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:38:24.362152851Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:38:24.362176109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:38:24.365051911Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T00:38:24.540673255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:38:24.544331483Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-04-21T00:38:24.544353002Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:38:24.662604881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:38:24.666070037Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-04-21T00:38:25.010911985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:47518"
level=info ts=2026-04-21T00:38:25.19668698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:38:25.20107227Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T00:38:25.563674324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:38:25.569729796Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T00:38:27.310136081Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:38:28.349473194Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:38:28.50699139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:38:28.510600468Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-04-21T00:38:28.879677033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:41468"
ts=2026-04-21T00:38:28.949008828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:51344"
level=info ts=2026-04-21T00:38:31.089594547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:38:31.093284657Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T00:38:31.267540724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:38:31.271917413Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-04-21T00:38:32.310382465Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:38:34.894509059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:38:34.901194496Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-21T00:38:37.151368095Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:38:37.310524619Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:38:38.220682721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:38:38.232250143Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T00:38:38.63432733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:38:38.637850567Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T00:38:42.273106618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:38:42.277030234Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-04-21T00:38:42.31015852Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:38:43.13194503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:38:43.135969829Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T00:38:44.104913562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:38:44.108716827Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:38:44.108742366Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:38:44.108768842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:38:44.111726586Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-04-21T00:38:47.310437203Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:38:48.81110595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:38:48.814963299Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T00:38:50.744612533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:38:50.750459449Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T00:38:51.481746159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:38:51.485184932Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T00:38:52.286341446Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:38:52.286383793Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T00:38:52.309656849Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-21T00:38:52.30967202Z caller=checkpoint.go:612 msg="starting checkpoint"
level=debug ts=2026-04-21T00:38:52.309693711Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:38:52.309717518Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:38:52.309738609Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:38:52.309846217Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000014
level=debug ts=2026-04-21T00:38:52.310020026Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=debug ts=2026-04-21T00:38:53.056860713Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:38:53.886214186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:38:53.893218899Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:38:53.89324397Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:38:53.89327753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:38:53.900076774Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-21T00:38:55.012285097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:41892"
level=info ts=2026-04-21T00:38:55.218348609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:38:55.22317812Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:38:56.113681962Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=warn ts=2026-04-21T00:38:57.309891436Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:38:59.916091497Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-21T00:39:02.310394605Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:39:03.240348714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:39:03.244082921Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T00:39:03.718636724Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=info ts=2026-04-21T00:39:05.533571228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:39:05.539246115Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-21T00:39:07.153574805Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:39:07.310492592Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:39:07.522407819Z caller=checkpoint.go:378 msg="writing series" size="55 kB"
level=info ts=2026-04-21T00:39:07.999747972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:39:08.00632737Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T00:39:10.072660382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:39:10.07924746Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:39:11.32544826Z caller=checkpoint.go:378 msg="writing series" size="688 kB"
level=warn ts=2026-04-21T00:39:12.310100903Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:39:12.395886055Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T00:39:12.397724803Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T00:39:13.248840405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:39:13.253085745Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-21T00:39:14.356433154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:39:14.360484265Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T00:39:15.128234112Z caller=checkpoint.go:378 msg="writing series" size="968 kB"
level=debug ts=2026-04-21T00:39:15.128262457Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=7
level=warn ts=2026-04-21T00:39:17.309732194Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:39:17.812922065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:39:17.816819383Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T00:39:18.932615258Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-21T00:39:18.932655602Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=1
level=info ts=2026-04-21T00:39:22.113074268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:39:22.11658699Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T00:39:22.309470596Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:39:22.309507388Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:39:22.309515955Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:39:22.309527618Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:39:22.733332555Z caller=checkpoint.go:378 msg="writing series" size="210 kB"
level=debug ts=2026-04-21T00:39:24.22577176Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:39:24.573950003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:39:24.578950982Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-04-21T00:39:25.013798321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:49236"
level=debug ts=2026-04-21T00:39:26.536153178Z caller=checkpoint.go:378 msg="writing series" size="124 kB"
level=warn ts=2026-04-21T00:39:27.3102631Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:39:28.952374661Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:34796"
level=info ts=2026-04-21T00:39:29.753381502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:39:29.760851096Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:39:30.338674687Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=warn ts=2026-04-21T00:39:32.309778628Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:39:34.142189321Z caller=checkpoint.go:378 msg="writing series" size="157 kB"
level=info ts=2026-04-21T00:39:34.672637552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:39:34.676333324Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T00:39:35.560689518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:39:35.564412122Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T00:39:35.744926688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:39:35.75188781Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-04-21T00:39:37.155713878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:39:37.31006605Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:39:37.968598551Z caller=checkpoint.go:378 msg="writing series" size="23 MB"
level=debug ts=2026-04-21T00:39:37.968644034Z caller=checkpoint.go:390 msg="flushing series" totalSize="24 MB" series=5
level=info ts=2026-04-21T00:39:41.513830354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:39:41.52006513Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T00:39:41.747650815Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=warn ts=2026-04-21T00:39:42.310320609Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:39:45.551819497Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-21T00:39:45.551844982Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=2
level=info ts=2026-04-21T00:39:46.457179984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:39:46.46134482Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-21T00:39:47.310275969Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:39:47.432603419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:39:47.437121993Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T00:39:48.102896644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:39:48.11074545Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:39:49.352781153Z caller=checkpoint.go:378 msg="writing series" size="138 kB"
level=info ts=2026-04-21T00:39:50.9171731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:39:50.921771209Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:39:52.286564169Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:39:52.286573763Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:39:52.310196622Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:39:52.310224696Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:39:52.310232773Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:39:52.31024232Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:39:53.155959579Z caller=checkpoint.go:378 msg="writing series" size="7.2 kB"
level=info ts=2026-04-21T00:39:54.295734392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:39:54.2998895Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T00:39:56.958106557Z caller=checkpoint.go:378 msg="writing series" size="78 kB"
level=warn ts=2026-04-21T00:39:57.30985322Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:39:57.667997087Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T00:39:58.954077036Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:52254"
level=info ts=2026-04-21T00:39:59.172796708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:39:59.176687837Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T00:40:00.761714213Z caller=checkpoint.go:378 msg="writing series" size="280 kB"
level=warn ts=2026-04-21T00:40:02.310043262Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:40:03.853207439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:40:03.857392041Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T00:40:04.294981552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:40:04.302528472Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T00:40:04.564135492Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=info ts=2026-04-21T00:40:04.654464295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:40:04.659218729Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-04-21T00:40:04.659242849Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:40:04.65927055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:40:04.662766393Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-04-21T00:40:07.157785775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:40:07.310393415Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:40:08.366730885Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=debug ts=2026-04-21T00:40:12.169849781Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-21T00:40:12.310362198Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:40:13.707798654Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T00:40:14.139994187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:40:14.143938714Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T00:40:15.708156634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:40:15.711574074Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T00:40:15.972241367Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-21T00:40:17.310162588Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:40:17.720117521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:40:17.72407579Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T00:40:18.913990546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:40:18.921686444Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T00:40:19.729125313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:40:19.73318736Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-21T00:40:19.775424025Z caller=checkpoint.go:378 msg="writing series" size="244 kB"
level=info ts=2026-04-21T00:40:21.07091508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:40:21.076580396Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T00:40:21.217967462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:40:21.221502663Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-21T00:40:22.310370033Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:40:22.31037877Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:40:22.3104026Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:40:22.310411376Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:40:23.578380814Z caller=checkpoint.go:378 msg="writing series" size="247 kB"
level=warn ts=2026-04-21T00:40:27.30975692Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:40:27.381204965Z caller=checkpoint.go:378 msg="writing series" size="269 kB"
level=debug ts=2026-04-21T00:40:27.381233768Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=11
level=debug ts=2026-04-21T00:40:31.184179957Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=debug ts=2026-04-21T00:40:31.348880229Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:40:32.309533739Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:40:32.516919866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:40:32.523587884Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T00:40:34.444363879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:40:34.449072822Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:40:34.986861141Z caller=checkpoint.go:378 msg="writing series" size="7.4 kB"
ts=2026-04-21T00:40:37.159740634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:40:37.309891307Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:40:38.789292439Z caller=checkpoint.go:378 msg="writing series" size="261 kB"
level=info ts=2026-04-21T00:40:39.710035508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:40:39.714123847Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T00:40:40.53105019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:40:40.534933665Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T00:40:40.964070769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:40:40.967573285Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-04-21T00:40:42.310508161Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:40:42.591894802Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=info ts=2026-04-21T00:40:44.356415647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:40:44.368916018Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T00:40:45.984143543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:40:45.988170072Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T00:40:46.395461137Z caller=checkpoint.go:378 msg="writing series" size="192 kB"
level=warn ts=2026-04-21T00:40:47.310290133Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:40:50.198130864Z caller=checkpoint.go:378 msg="writing series" size="105 kB"
level=info ts=2026-04-21T00:40:52.286423533Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:40:52.286448436Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:40:52.309814632Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:40:52.309833386Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:40:52.309847515Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:40:52.309857801Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T00:40:52.754534779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:40:52.758433675Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T00:40:54.000985018Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-04-21T00:40:54.821468918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:40:54.829469008Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:40:54.829499704Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:40:54.829532983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:40:54.835662472Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T00:40:55.137752142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:40:55.141860246Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-04-21T00:40:57.310287418Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:40:57.80421067Z caller=checkpoint.go:378 msg="writing series" size="526 kB"
level=debug ts=2026-04-21T00:40:57.804234973Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=8
level=info ts=2026-04-21T00:40:59.481361862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:40:59.484613865Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T00:41:01.615508179Z caller=checkpoint.go:378 msg="writing series" size="12 MB"
level=debug ts=2026-04-21T00:41:01.615539922Z caller=checkpoint.go:390 msg="flushing series" totalSize="12 MB" series=1
level=debug ts=2026-04-21T00:41:01.670385511Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T00:41:02.310357574Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:41:03.446561183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:41:03.454166769Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T00:41:03.539476329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:41:03.543118599Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T00:41:05.408944792Z caller=checkpoint.go:378 msg="writing series" size="657 B"
level=debug ts=2026-04-21T00:41:06.310974932Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T00:41:07.161452481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:41:07.30986977Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:41:07.588933409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:41:07.595823196Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T00:41:09.212686365Z caller=checkpoint.go:378 msg="writing series" size="532 kB"
level=info ts=2026-04-21T00:41:09.995323918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:41:09.998895052Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T00:41:11.372172622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:41:11.375536076Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-21T00:41:12.310181798Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:41:13.014512422Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=info ts=2026-04-21T00:41:13.437996377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:41:13.441648448Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T00:41:15.300036373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:41:15.305770655Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-21T00:41:16.817352048Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=warn ts=2026-04-21T00:41:17.310261415Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:41:20.620702592Z caller=checkpoint.go:378 msg="writing series" size="162 kB"
level=info ts=2026-04-21T00:41:21.594812403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:41:21.598832802Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:41:22.309940751Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:41:22.309973385Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:41:22.309982316Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:41:22.309989233Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:41:24.42369852Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=warn ts=2026-04-21T00:41:27.31042691Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:41:28.226115709Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
ts=2026-04-21T00:41:28.89012943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:57162"
level=debug ts=2026-04-21T00:41:32.029789602Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=warn ts=2026-04-21T00:41:32.309652821Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:41:35.174246944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:41:35.180430254Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T00:41:35.350758149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:41:35.358882711Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T00:41:35.76356587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:41:35.76791865Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T00:41:35.832148712Z caller=checkpoint.go:378 msg="writing series" size="817 B"
level=debug ts=2026-04-21T00:41:35.835378243Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:41:35.835423896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:41:35.840592476Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T00:41:36.136506247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="446 B" avg_comp="446 B" total_uncomp="36 kB" avg_uncomp="36 kB" idle=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:41:36.140219622Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-04-21T00:41:37.16331141Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:41:37.31032183Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:41:37.648090156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:41:37.651698026Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T00:41:39.427351088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:41:39.430953519Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T00:41:39.63417134Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=info ts=2026-04-21T00:41:40.098930464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:41:40.102388092Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-04-21T00:41:42.310060565Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:41:43.437580988Z caller=checkpoint.go:378 msg="writing series" size="841 kB"
level=debug ts=2026-04-21T00:41:43.43760655Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=11
level=info ts=2026-04-21T00:41:44.609885986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:41:44.616936508Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T00:41:46.358655652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:41:46.366688337Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T00:41:46.486269953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:41:46.49127526Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T00:41:47.240461029Z caller=checkpoint.go:378 msg="writing series" size="173 kB"
level=warn ts=2026-04-21T00:41:47.310392178Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:41:51.04354477Z caller=checkpoint.go:378 msg="writing series" size="518 kB"
level=info ts=2026-04-21T00:41:51.855214767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:41:51.858687783Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-21T00:41:52.285939218Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:41:52.28700297Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T00:41:52.30943945Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:41:52.309466579Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:41:52.309475067Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:41:52.309508816Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:41:52.787727499Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T00:41:52.918129872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:41:52.922112497Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-21T00:41:54.846250816Z caller=checkpoint.go:378 msg="writing series" size="338 kB"
ts=2026-04-21T00:41:55.023373434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:44304"
level=info ts=2026-04-21T00:41:56.417905947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:41:56.421828242Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-04-21T00:41:57.310131137Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:41:58.31506965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:41:58.318696927Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T00:41:58.560280685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:41:58.564019942Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T00:41:58.648304561Z caller=checkpoint.go:378 msg="writing series" size="94 kB"
level=debug ts=2026-04-21T00:41:58.6483281Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=4
ts=2026-04-21T00:41:58.961520151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:60298"
level=warn ts=2026-04-21T00:42:02.309512722Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:42:02.451902462Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=debug ts=2026-04-21T00:42:06.254378481Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
ts=2026-04-21T00:42:07.164914Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:42:07.310108132Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:42:08.209614651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:42:08.21376285Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:42:08.213783377Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:42:08.213806675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:42:08.217161267Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:42:09.111189037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:42:09.11531623Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T00:42:10.03420928Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:42:10.047226988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:42:10.050701532Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T00:42:10.056928043Z caller=checkpoint.go:378 msg="writing series" size="435 kB"
level=warn ts=2026-04-21T00:42:12.310191393Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:42:13.859683412Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=info ts=2026-04-21T00:42:15.098131567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:42:15.103210369Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:42:15.103232816Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:42:15.103257744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:42:15.107909155Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-21T00:42:17.309548092Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:42:17.663132035Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-04-21T00:42:20.374470941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:42:20.379559153Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-21T00:42:20.639324071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:42:20.644348369Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T00:42:21.465827163Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-04-21T00:42:22.310297871Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:42:22.310294458Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:42:22.310322728Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:42:22.310331824Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T00:42:23.143205009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:42:23.147252114Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:42:24.158205901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:42:24.166032654Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:42:25.268018761Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=warn ts=2026-04-21T00:42:27.310454329Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:42:27.955638819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:42:27.96070002Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T00:42:27.964745878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:42:27.96859129Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-21T00:42:28.894169971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:38142"
ts=2026-04-21T00:42:28.963993052Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:42696"
level=info ts=2026-04-21T00:42:28.981663745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:42:28.985101238Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T00:42:29.071263131Z caller=checkpoint.go:378 msg="writing series" size="162 kB"
level=info ts=2026-04-21T00:42:30.25735945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:42:30.264030142Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T00:42:30.918951425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:42:30.92254173Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T00:42:30.939444759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:42:30.94272364Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-21T00:42:32.309516155Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:42:32.874184399Z caller=checkpoint.go:378 msg="writing series" size="8.2 kB"
level=debug ts=2026-04-21T00:42:35.71160971Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:42:36.677247872Z caller=checkpoint.go:378 msg="writing series" size="113 kB"
level=debug ts=2026-04-21T00:42:36.677273587Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
ts=2026-04-21T00:42:37.167225166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:42:37.309552969Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:42:37.319676182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=info ts=2026-04-21T00:42:37.321888022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:42:37.324052943Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-04-21T00:42:37.32831209Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T00:42:39.971212993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:42:39.974613247Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T00:42:40.481199932Z caller=checkpoint.go:378 msg="writing series" size="2.4 MB"
level=debug ts=2026-04-21T00:42:40.481224873Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.4 MB" series=1
level=warn ts=2026-04-21T00:42:42.309786008Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:42:44.283066291Z caller=checkpoint.go:378 msg="writing series" size="392 kB"
level=info ts=2026-04-21T00:42:46.848182867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:42:46.853037427Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-21T00:42:47.310230579Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:42:47.385551317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:42:47.389345602Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T00:42:48.085015836Z caller=checkpoint.go:378 msg="writing series" size="869 kB"
level=debug ts=2026-04-21T00:42:48.085041747Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=2
level=info ts=2026-04-21T00:42:49.56445842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:42:49.568336724Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T00:42:50.700443417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:42:50.705927133Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T00:42:51.88828632Z caller=checkpoint.go:378 msg="writing series" size="5.0 kB"
level=debug ts=2026-04-21T00:42:52.286332573Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:42:52.286368388Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T00:42:52.309576584Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:42:52.309605799Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:42:52.309615335Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:42:52.309638418Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:42:54.762126494Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T00:42:55.691134458Z caller=checkpoint.go:378 msg="writing series" size="336 kB"
level=warn ts=2026-04-21T00:42:57.310381144Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:42:58.555688656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:42:58.560908657Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-04-21T00:42:58.896444779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:34808"
level=debug ts=2026-04-21T00:42:59.494107775Z caller=checkpoint.go:378 msg="writing series" size="943 kB"
level=debug ts=2026-04-21T00:42:59.494131623Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=3
level=info ts=2026-04-21T00:43:02.198268702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:43:02.201829688Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-21T00:43:02.310116507Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:43:02.750923047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:43:02.754536741Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T00:43:03.29629349Z caller=checkpoint.go:378 msg="writing series" size="5.5 kB"
level=info ts=2026-04-21T00:43:04.32075112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:43:04.326094425Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T00:43:07.099786006Z caller=checkpoint.go:378 msg="writing series" size="437 kB"
ts=2026-04-21T00:43:07.169445104Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:43:07.311289533Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:43:07.909380571Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:43:10.696673426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:43:10.700741316Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T00:43:10.902332451Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-21T00:43:12.30951063Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:43:12.627147429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:43:12.631462393Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:43:12.631482953Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:43:12.631507973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:43:12.635129589Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T00:43:13.163668464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:43:13.167239295Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T00:43:14.705089725Z caller=checkpoint.go:378 msg="writing series" size="193 kB"
level=info ts=2026-04-21T00:43:16.935134242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:43:16.938891438Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-21T00:43:17.309686962Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:43:17.400005489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:43:17.40348939Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-21T00:43:18.508221255Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=info ts=2026-04-21T00:43:20.567340543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:43:20.574366802Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:43:22.310360105Z caller=checkpoint.go:390 msg="flushing series" totalSize="650 kB" series=5
level=warn ts=2026-04-21T00:43:22.310387459Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:43:22.310408256Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:43:22.310431605Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:43:22.310440025Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T00:43:22.3159435Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000014.tmp new=/var/loki/wal/checkpoint.000014
level=info ts=2026-04-21T00:43:22.319985267Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.010005902s
level=info ts=2026-04-21T00:43:26.1728875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:43:26.180816607Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T00:43:26.65014026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:43:26.65725945Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:43:26.657285498Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:43:26.657318924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:43:26.664608358Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T00:43:27.310457103Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:43:28.898156091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:47122"
ts=2026-04-21T00:43:28.968497728Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:43952"
level=info ts=2026-04-21T00:43:29.66858483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:43:29.67352098Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T00:43:32.310201485Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:43:32.446552974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:43:32.450290759Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T00:43:32.588612275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:43:32.592940194Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T00:43:35.147921709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:43:35.151573671Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T00:43:36.864687452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:43:36.868582456Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-04-21T00:43:37.171329714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:43:37.309728288Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:43:39.078336994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:43:39.085458695Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T00:43:40.911420909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:43:40.915107001Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T00:43:42.229866241Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:43:42.310128167Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:43:44.519964966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:43:44.52474873Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:43:46.683726759Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T00:43:46.685473462Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T00:43:47.309848081Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:43:51.707282648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:43:51.712460962Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T00:43:52.286536368Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:43:52.286974209Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:43:52.309333264Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-21T00:43:52.309652233Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000015
level=info ts=2026-04-21T00:43:52.309687919Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:43:52.30970854Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:43:52.309716597Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:43:52.309714051Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:43:52.309859877Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=debug ts=2026-04-21T00:43:56.0603231Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=warn ts=2026-04-21T00:43:57.310294109Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:43:57.885070659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:43:57.890486932Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T00:43:58.224307945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:43:58.228813877Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-04-21T00:43:58.899928686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:40066"
level=info ts=2026-04-21T00:43:58.985682947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:43:58.989351136Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-21T00:43:59.810779845Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=info ts=2026-04-21T00:44:02.030829864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:44:02.034734754Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-21T00:44:02.309536978Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:44:02.912454201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:44:02.917885041Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T00:44:03.051141485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:44:03.054905143Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T00:44:03.559975818Z caller=checkpoint.go:378 msg="writing series" size="168 kB"
level=info ts=2026-04-21T00:44:06.318679493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:44:06.323001052Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-21T00:44:07.172965266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:44:07.310068311Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:44:07.31012208Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
level=info ts=2026-04-21T00:44:07.969884415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:44:07.977150059Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:44:10.116710599Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:44:11.061035763Z caller=checkpoint.go:378 msg="writing series" size="689 kB"
level=warn ts=2026-04-21T00:44:12.310405516Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:44:12.884820822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:44:12.888600453Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T00:44:13.750208106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:44:13.753938839Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:44:13.75396226Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:44:13.753987264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:44:13.757456233Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T00:44:13.931082637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:44:13.938167936Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:44:14.811081693Z caller=checkpoint.go:378 msg="writing series" size="970 kB"
level=debug ts=2026-04-21T00:44:14.811108392Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=7
level=info ts=2026-04-21T00:44:15.656768479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:44:15.660537203Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T00:44:15.922509515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:44:15.926147519Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-04-21T00:44:17.309866117Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:44:18.561914605Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-21T00:44:18.561941041Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=1
level=info ts=2026-04-21T00:44:22.310345153Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:44:22.310393163Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-21T00:44:22.310387753Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:44:22.310405642Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:44:22.310506854Z caller=checkpoint.go:378 msg="writing series" size="213 kB"
level=info ts=2026-04-21T00:44:22.38567848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:44:22.392117307Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T00:44:25.471356511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:44:25.475021508Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T00:44:26.060893882Z caller=checkpoint.go:378 msg="writing series" size="132 kB"
level=warn ts=2026-04-21T00:44:27.310232263Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:44:29.810275218Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=info ts=2026-04-21T00:44:30.73048362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:44:30.736228238Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-21T00:44:31.427287508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:44:31.430890382Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:44:31.430911401Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:44:31.430934377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:44:31.433786565Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-21T00:44:32.310257805Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:44:33.560712535Z caller=checkpoint.go:378 msg="writing series" size="165 kB"
ts=2026-04-21T00:44:37.174980512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:44:37.309912252Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:44:37.332929165Z caller=checkpoint.go:378 msg="writing series" size="23 MB"
level=debug ts=2026-04-21T00:44:37.332958036Z caller=checkpoint.go:390 msg="flushing series" totalSize="24 MB" series=5
level=info ts=2026-04-21T00:44:39.693864957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:44:39.697784107Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:44:41.005329112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:44:41.009163151Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-21T00:44:41.060294836Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=debug ts=2026-04-21T00:44:41.205148172Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:44:42.26301181Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T00:44:42.310372845Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:44:44.695523091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:44:44.698978063Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-04-21T00:44:44.699002763Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:44:44.699025703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:44:44.702021307Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T00:44:44.812315214Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB"
level=debug ts=2026-04-21T00:44:44.812340675Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-04-21T00:44:44.947240866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:44:44.950949793Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T00:44:45.207312489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:44:45.21078517Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:44:45.210808275Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:44:45.210837884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:44:45.213742066Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-21T00:44:47.309850861Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:44:48.080420378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:44:48.085933904Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-21T00:44:48.56058438Z caller=checkpoint.go:378 msg="writing series" size="169 kB"
level=info ts=2026-04-21T00:44:49.837718704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:44:49.859832399Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T00:44:52.208605926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:44:52.212147027Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T00:44:52.285890289Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:44:52.286948811Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T00:44:52.310394459Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:44:52.310425933Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:44:52.310434576Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:44:52.31046895Z caller=checkpoint.go:378 msg="writing series" size="7.2 kB"
level=warn ts=2026-04-21T00:44:52.310470443Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:44:52.411978807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:44:52.419334237Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T00:44:54.487963611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:44:54.491918328Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:44:54.491939928Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:44:54.491964257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:44:54.495138851Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T00:44:55.331274778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:44:55.335115886Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:44:55.335152288Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:44:55.335186227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:44:55.338301337Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T00:44:56.060338837Z caller=checkpoint.go:378 msg="writing series" size="78 kB"
level=warn ts=2026-04-21T00:44:57.31000364Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:44:57.339456606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:44:57.345216309Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T00:44:58.200863209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:44:58.204828974Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:44:58.20485466Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:44:58.204886747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:44:58.208165386Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-04-21T00:44:58.974785475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:54072"
level=debug ts=2026-04-21T00:44:59.810204166Z caller=checkpoint.go:378 msg="writing series" size="280 kB"
level=info ts=2026-04-21T00:45:00.821770185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:45:00.829174596Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:45:00.829205155Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:45:00.829237509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:45:00.832727726Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T00:45:01.924820257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:45:01.928542869Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-21T00:45:02.309651781Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:45:03.560411493Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=info ts=2026-04-21T00:45:03.755093941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:45:03.759011945Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T00:45:06.749785112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:45:06.753847688Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T00:45:07.124109428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:45:07.13041823Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-21T00:45:07.176685174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:45:07.310360648Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:45:07.310424649Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=info ts=2026-04-21T00:45:07.729440611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:45:07.732960096Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:45:07.732982191Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:45:07.733004546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:45:07.735902328Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T00:45:09.277101883Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:45:09.856746201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:45:09.860129383Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T00:45:09.928118774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:45:09.934746789Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:45:11.060338932Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=warn ts=2026-04-21T00:45:12.309870621Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:45:13.1831421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:45:13.186807486Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T00:45:14.215839361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:45:14.219468126Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T00:45:14.810343292Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-21T00:45:17.310075687Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:45:18.09758947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:45:18.101516163Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T00:45:18.506411754Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T00:45:18.560728416Z caller=checkpoint.go:378 msg="writing series" size="257 kB"
level=info ts=2026-04-21T00:45:19.137482315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:45:19.141908718Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-21T00:45:20.839387509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:45:20.843227153Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T00:45:22.309581663Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:45:22.309615794Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:45:22.30963807Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:45:22.309678262Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:45:22.310859624Z caller=checkpoint.go:378 msg="writing series" size="274 kB"
level=debug ts=2026-04-21T00:45:22.310869498Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-04-21T00:45:22.472828604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:45:22.476346609Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T00:45:26.060438679Z caller=checkpoint.go:378 msg="writing series" size="269 kB"
level=info ts=2026-04-21T00:45:26.270755579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:45:26.274524267Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-21T00:45:27.310261093Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:45:28.976410086Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:49638"
level=debug ts=2026-04-21T00:45:29.81031559Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=info ts=2026-04-21T00:45:30.093481426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:45:30.097764051Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:45:30.105814136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:45:30.108968691Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-21T00:45:30.706440398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:45:30.710933243Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T00:45:30.874238053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:45:30.881490159Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-21T00:45:32.309495798Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:45:33.559979701Z caller=checkpoint.go:378 msg="writing series" size="7.4 kB"
ts=2026-04-21T00:45:37.178490601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:45:37.309519388Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:45:37.310781861Z caller=checkpoint.go:378 msg="writing series" size="312 kB"
level=debug ts=2026-04-21T00:45:38.668783362Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:45:41.060616223Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=warn ts=2026-04-21T00:45:42.310279736Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:45:42.708674151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:45:42.714749171Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T00:45:43.564211965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:45:43.568435516Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T00:45:44.249306261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:45:44.252973209Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T00:45:44.810418909Z caller=checkpoint.go:378 msg="writing series" size="192 kB"
level=info ts=2026-04-21T00:45:45.330656561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:45:45.334267342Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T00:45:46.598054157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:45:46.601711504Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-04-21T00:45:47.309757726Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:45:47.82130551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:45:47.825155188Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T00:45:48.560883152Z caller=checkpoint.go:378 msg="writing series" size="105 kB"
level=info ts=2026-04-21T00:45:50.987763985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:45:50.994190109Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T00:45:51.723450816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:45:51.730224063Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:45:51.73024742Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:45:51.730277772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:45:51.736654756Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T00:45:52.286510212Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:45:52.286905148Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=debug ts=2026-04-21T00:45:52.288180525Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-21T00:45:52.288220319Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974146 n=1
level=debug ts=2026-04-21T00:45:52.288264038Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974146 n=1
level=info ts=2026-04-21T00:45:52.294245316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:45:52.297600894Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-21T00:45:52.309693557Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:45:52.309707845Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:45:52.30973169Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:45:52.309740754Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:45:52.310833485Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=debug ts=2026-04-21T00:45:52.310848903Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-04-21T00:45:53.235373257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:45:53.241751847Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T00:45:54.725958322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:45:54.730926094Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-04-21T00:45:55.038696609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:39246"
level=debug ts=2026-04-21T00:45:56.060130956Z caller=checkpoint.go:378 msg="writing series" size="452 kB"
level=warn ts=2026-04-21T00:45:57.309777776Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:45:58.224937136Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T00:45:59.817693478Z caller=checkpoint.go:378 msg="writing series" size="13 MB"
level=debug ts=2026-04-21T00:45:59.817719991Z caller=checkpoint.go:390 msg="flushing series" totalSize="13 MB" series=2
level=warn ts=2026-04-21T00:46:02.310344778Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:46:03.559914135Z caller=checkpoint.go:378 msg="writing series" size="657 B"
level=debug ts=2026-04-21T00:46:03.927097255Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:46:05.518418842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:46:05.529321287Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-21T00:46:07.180488253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:46:07.309915585Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:46:07.310360469Z caller=checkpoint.go:378 msg="writing series" size="544 kB"
level=debug ts=2026-04-21T00:46:11.059936308Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-21T00:46:12.310511385Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:46:12.371653398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:46:12.376026398Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-21T00:46:14.810820047Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=info ts=2026-04-21T00:46:15.250231718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:46:15.253856372Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-21T00:46:16.621680556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:46:16.626251441Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T00:46:17.310327892Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:46:18.560313559Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=info ts=2026-04-21T00:46:21.334985829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:46:21.339591593Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-21T00:46:22.309973807Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:46:22.309985854Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:46:22.310007173Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:46:22.310015674Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:46:22.310072123Z caller=checkpoint.go:378 msg="writing series" size="169 kB"
level=info ts=2026-04-21T00:46:23.527047418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:46:23.531361768Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T00:46:24.272117933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:46:24.275839747Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T00:46:26.060313683Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=info ts=2026-04-21T00:46:26.506947876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:46:26.511936718Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-21T00:46:27.309879565Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:46:28.275374596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:46:28.282776762Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T00:46:28.994366376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:46:28.997984228Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T00:46:29.405282679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:46:29.408818137Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T00:46:29.608435924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:46:29.612502284Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:46:29.684000895Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:46:29.810299202Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
level=info ts=2026-04-21T00:46:31.564353531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:46:31.568199906Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-04-21T00:46:32.310364858Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:46:33.56005625Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-04-21T00:46:33.642479779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:46:33.649075467Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T00:46:36.962637414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:46:36.967307174Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-04-21T00:46:37.182732182Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:46:37.309748421Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:46:37.31078924Z caller=checkpoint.go:378 msg="writing series" size="817 B"
level=info ts=2026-04-21T00:46:38.63967224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:46:38.64483809Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T00:46:40.76656581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:46:40.773358906Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:46:41.060212159Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=debug ts=2026-04-21T00:46:41.500721361Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T00:46:42.310266393Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:46:44.811174887Z caller=checkpoint.go:378 msg="writing series" size="841 kB"
level=debug ts=2026-04-21T00:46:44.811201312Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=12
level=warn ts=2026-04-21T00:46:47.30980792Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:46:48.560202023Z caller=checkpoint.go:378 msg="writing series" size="200 kB"
level=info ts=2026-04-21T00:46:50.188226241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:46:50.193564743Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T00:46:51.321389572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:46:51.325228719Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T00:46:52.286406693Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:46:52.286834062Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=warn ts=2026-04-21T00:46:52.310295477Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:46:52.310293518Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:46:52.310326134Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:46:52.310334686Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:46:52.310865725Z caller=checkpoint.go:378 msg="writing series" size="578 kB"
level=debug ts=2026-04-21T00:46:55.307145804Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:46:56.060975368Z caller=checkpoint.go:378 msg="writing series" size="355 kB"
level=debug ts=2026-04-21T00:46:56.061002436Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=3
level=warn ts=2026-04-21T00:46:57.309943319Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:46:58.909958342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:41932"
level=debug ts=2026-04-21T00:46:59.810239508Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=warn ts=2026-04-21T00:47:02.310483768Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:47:03.559914479Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=info ts=2026-04-21T00:47:05.507765836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:47:05.515605159Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-04-21T00:47:05.515652072Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:47:05.515679022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:47:05.51935611Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-04-21T00:47:07.184882772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=debug ts=2026-04-21T00:47:07.310281773Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-21T00:47:07.310288733Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:47:10.233142424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:47:10.239439329Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T00:47:11.060573219Z caller=checkpoint.go:378 msg="writing series" size="435 kB"
level=warn ts=2026-04-21T00:47:12.310289917Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:47:12.619916318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:47:12.623911895Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-21T00:47:13.347383511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:47:13.354506127Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T00:47:14.066386953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:47:14.069889356Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T00:47:14.810485196Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=warn ts=2026-04-21T00:47:17.309757087Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:47:17.532067417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:47:17.535613265Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T00:47:18.37907481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:47:18.382807097Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T00:47:18.52838571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:47:18.532469691Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-21T00:47:18.560605901Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-04-21T00:47:19.589844352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:47:19.596437569Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:47:19.596460968Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:47:19.596486245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:47:19.602178967Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T00:47:20.015588439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:47:20.020595752Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T00:47:20.077607888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:47:20.081415518Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T00:47:20.812289771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:47:20.816186885Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T00:47:21.747920904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:47:21.751695959Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T00:47:22.021367987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:47:22.024747004Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-04-21T00:47:22.310469762Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:47:22.310462489Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-04-21T00:47:22.31046565Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:47:22.310491271Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:47:22.310499886Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:47:23.444307827Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T00:47:25.044298006Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:42140"
level=info ts=2026-04-21T00:47:25.476488481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:47:25.480901049Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:47:26.060711958Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=warn ts=2026-04-21T00:47:27.310277123Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:47:29.810229005Z caller=checkpoint.go:378 msg="writing series" size="219 kB"
level=debug ts=2026-04-21T00:47:29.810253293Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-04-21T00:47:31.11910824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:47:31.122783585Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-04-21T00:47:32.310234431Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:47:33.559969696Z caller=checkpoint.go:378 msg="writing series" size="8.2 kB"
level=info ts=2026-04-21T00:47:34.80885232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:47:34.812465016Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T00:47:35.383748623Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T00:47:35.385281988Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T00:47:37.18649009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:47:37.310218158Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:47:37.310329824Z caller=checkpoint.go:378 msg="writing series" size="116 kB"
level=info ts=2026-04-21T00:47:38.031996122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:47:38.038338025Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T00:47:39.42617077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:47:39.43389368Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T00:47:40.16399718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:47:40.168023508Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-21T00:47:41.062040158Z caller=checkpoint.go:378 msg="writing series" size="2.5 MB"
level=debug ts=2026-04-21T00:47:41.062065065Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.6 MB" series=3
level=warn ts=2026-04-21T00:47:42.309768326Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:47:44.810701581Z caller=checkpoint.go:378 msg="writing series" size="497 kB"
level=info ts=2026-04-21T00:47:45.679957258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:47:45.685060308Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-21T00:47:47.310407226Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:47:48.561172956Z caller=checkpoint.go:378 msg="writing series" size="869 kB"
level=debug ts=2026-04-21T00:47:48.56119682Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-04-21T00:47:52.286548858Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:47:52.286988469Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:47:52.31042364Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:47:52.310421755Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:47:52.310448783Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:47:52.310459408Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:47:52.310487182Z caller=checkpoint.go:378 msg="writing series" size="5.2 kB"
level=debug ts=2026-04-21T00:47:53.176888867Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T00:47:55.045513117Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:42348"
level=info ts=2026-04-21T00:47:55.631934494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:47:55.636077318Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:47:55.636110707Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:47:55.636147955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:47:55.639752837Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-21T00:47:56.061056148Z caller=checkpoint.go:378 msg="writing series" size="336 kB"
level=info ts=2026-04-21T00:47:57.041686801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:47:57.049294356Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-21T00:47:57.309882647Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:47:59.811464066Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-21T00:47:59.811487959Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=3
level=info ts=2026-04-21T00:48:00.806341022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:48:00.81000303Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-21T00:48:02.309668434Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:48:02.840912479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:48:02.845979561Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-21T00:48:03.560213534Z caller=checkpoint.go:378 msg="writing series" size="5.5 kB"
level=info ts=2026-04-21T00:48:06.723583427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:48:06.730070492Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-21T00:48:07.188102087Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:48:07.309954804Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:48:07.310121633Z caller=checkpoint.go:378 msg="writing series" size="437 kB"
level=info ts=2026-04-21T00:48:10.19256996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:48:10.197032908Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T00:48:10.898519635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:48:10.902046361Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T00:48:11.060372268Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-04-21T00:48:11.441306942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:48:11.444701663Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-04-21T00:48:12.309900782Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:48:13.569997774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:48:13.574250512Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:48:13.686823747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:48:13.690426876Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-21T00:48:14.809990845Z caller=checkpoint.go:378 msg="writing series" size="194 kB"
level=info ts=2026-04-21T00:48:14.902882199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:48:14.906385132Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T00:48:15.691481223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:48:15.694973689Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T00:48:15.717613855Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T00:48:17.309755309Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:48:18.415125422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:48:18.418781868Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T00:48:18.560764826Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=info ts=2026-04-21T00:48:20.430475003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:48:20.435704389Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T00:48:20.539087031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:48:20.543177615Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-21T00:48:21.224055786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:48:21.231075068Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T00:48:22.309616907Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:48:22.309652143Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:48:22.309682019Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:48:22.30969307Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:48:22.31074357Z caller=checkpoint.go:390 msg="flushing series" totalSize="650 kB" series=5
level=info ts=2026-04-21T00:48:22.318675075Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000015.tmp new=/var/loki/wal/checkpoint.000015
level=info ts=2026-04-21T00:48:22.322608628Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.012790392s
level=info ts=2026-04-21T00:48:25.390612267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:48:25.394286567Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-04-21T00:48:27.309515192Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:48:29.084322248Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:48:32.310364845Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:48:32.357525749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:48:32.361486254Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T00:48:36.790286931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:48:36.795744029Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:48:36.795768948Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:48:36.795795668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:48:36.80070912Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-04-21T00:48:37.190505765Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:48:37.309520898Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:48:38.119855339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:48:38.126906114Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T00:48:40.859193251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:48:40.862979916Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-04-21T00:48:42.309581996Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:48:45.514196623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:48:45.520344185Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:48:45.520368841Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:48:45.52039532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:48:45.524947731Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-21T00:48:47.310343143Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:48:47.776596119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:48:47.782576093Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T00:48:51.176687603Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T00:48:52.286150658Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:48:52.287195699Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-21T00:48:52.309523956Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:48:52.309514023Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-21T00:48:52.309530912Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:48:52.309551786Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:48:52.309560636Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T00:48:52.30969943Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000016
level=debug ts=2026-04-21T00:48:52.30988076Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=info ts=2026-04-21T00:48:53.092295262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:48:53.095992722Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T00:48:54.318773716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:48:54.322920833Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:48:54.322949334Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:48:54.32298429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:48:54.326524387Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:48:54.452765207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:48:54.456218019Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T00:48:56.060334218Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-04-21T00:48:56.20602318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:48:56.209745227Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T00:48:56.647158254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:48:56.652456011Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-21T00:48:57.310288336Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:48:59.810327225Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=debug ts=2026-04-21T00:49:00.231447775Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:49:02.310233365Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:49:02.35428551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:49:02.35929341Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T00:49:03.439039888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:49:03.442937351Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T00:49:03.560620841Z caller=checkpoint.go:378 msg="writing series" size="219 kB"
level=info ts=2026-04-21T00:49:04.951584703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:49:04.958641836Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T00:49:05.257999146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:49:05.262363166Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-04-21T00:49:07.192504676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:49:07.30998143Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:49:07.310030403Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=info ts=2026-04-21T00:49:08.280841894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:49:08.284743085Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T00:49:09.808609231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:49:09.812496327Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T00:49:10.529528572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:49:10.533772077Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-21T00:49:11.060876878Z caller=checkpoint.go:378 msg="writing series" size="689 kB"
level=warn ts=2026-04-21T00:49:12.309719422Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:49:13.40839009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:49:13.412236637Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T00:49:13.750845331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T00:49:13.75192416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:49:13.754385375Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:49:13.756724379Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T00:49:14.811442272Z caller=checkpoint.go:378 msg="writing series" size="972 kB"
level=debug ts=2026-04-21T00:49:14.811465652Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.0 MB" series=7
level=info ts=2026-04-21T00:49:16.863413875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:49:16.870844263Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-21T00:49:17.309765074Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:49:17.51305852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:49:17.516563706Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T00:49:18.560782134Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB"
level=debug ts=2026-04-21T00:49:18.560806674Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=1
level=info ts=2026-04-21T00:49:19.497904881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:49:19.505917448Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T00:49:21.137446826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:49:21.141116115Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-21T00:49:22.309709088Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:49:22.309711386Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:49:22.309736741Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:49:22.309745885Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:49:22.31096757Z caller=checkpoint.go:378 msg="writing series" size="216 kB"
ts=2026-04-21T00:49:25.051725883Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:47856"
level=debug ts=2026-04-21T00:49:26.060472585Z caller=checkpoint.go:378 msg="writing series" size="140 kB"
level=warn ts=2026-04-21T00:49:27.309525959Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:49:28.919109625Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:45036"
ts=2026-04-21T00:49:28.990703065Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:34112"
level=debug ts=2026-04-21T00:49:29.810781552Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=debug ts=2026-04-21T00:49:32.186776476Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:49:32.310121838Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:49:33.560424947Z caller=checkpoint.go:378 msg="writing series" size="174 kB"
ts=2026-04-21T00:49:37.194318858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:49:37.310468497Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:49:37.335322775Z caller=checkpoint.go:378 msg="writing series" size="23 MB"
level=debug ts=2026-04-21T00:49:37.335350742Z caller=checkpoint.go:390 msg="flushing series" totalSize="24 MB" series=5
level=info ts=2026-04-21T00:49:37.547351408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:49:37.557876321Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T00:49:39.876614008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:49:39.880168982Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T00:49:41.060649109Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=debug ts=2026-04-21T00:49:41.138280432Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T00:49:42.309701288Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:49:43.297662756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:49:43.301714566Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:49:43.671424621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:49:43.675885587Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:49:43.675908474Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:49:43.67593325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:49:43.679679285Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T00:49:44.812320472Z caller=checkpoint.go:378 msg="writing series" size="1.5 MB"
level=debug ts=2026-04-21T00:49:44.812344921Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=2
level=info ts=2026-04-21T00:49:46.947154943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:49:46.951748649Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-04-21T00:49:47.309978976Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:49:48.560753342Z caller=checkpoint.go:378 msg="writing series" size="198 kB"
level=info ts=2026-04-21T00:49:50.095368369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:49:50.099023424Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T00:49:50.795326272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:49:50.798975359Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T00:49:51.393913304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:49:51.400799099Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T00:49:52.286962568Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:49:52.287434838Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:49:52.309650867Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:49:52.309683911Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:49:52.309697122Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:49:52.309727032Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:49:52.310812191Z caller=checkpoint.go:378 msg="writing series" size="7.2 kB"
level=info ts=2026-04-21T00:49:54.867575964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:49:54.871152315Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T00:49:55.940077187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:49:55.94376535Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T00:49:56.06009669Z caller=checkpoint.go:378 msg="writing series" size="78 kB"
level=info ts=2026-04-21T00:49:56.069205733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:49:56.07344518Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-21T00:49:56.416760947Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:49:57.309881817Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:49:58.993081512Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:58450"
level=info ts=2026-04-21T00:49:59.548138877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:49:59.55304877Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T00:49:59.810936655Z caller=checkpoint.go:378 msg="writing series" size="280 kB"
level=info ts=2026-04-21T00:50:00.625561866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:50:00.62957116Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T00:50:00.858681269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:50:00.864789399Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-21T00:50:02.310492522Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:50:03.560317446Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=info ts=2026-04-21T00:50:05.108165914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:50:05.11387973Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T00:50:06.310328354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:50:06.314514632Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-04-21T00:50:07.196569726Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:50:07.30968534Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:50:07.310820042Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=info ts=2026-04-21T00:50:08.944662927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:50:08.949009436Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-21T00:50:10.044044332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:50:10.051353603Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:50:10.051376447Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:50:10.051419268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:50:10.058611811Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:50:11.05998481Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=warn ts=2026-04-21T00:50:12.310380342Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:50:12.547896846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:50:12.551951274Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T00:50:14.810856026Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-21T00:50:17.310196268Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:50:18.56041132Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=info ts=2026-04-21T00:50:22.309558666Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:50:22.309587841Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:50:22.309595869Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:50:22.309562216Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:50:22.310748548Z caller=checkpoint.go:378 msg="writing series" size="262 kB"
level=debug ts=2026-04-21T00:50:23.550019381Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T00:50:24.085581348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:50:24.094145342Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T00:50:24.494280862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:50:24.498509608Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T00:50:24.800010468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:50:24.803615977Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:50:24.803658098Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:50:24.803685484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:50:24.806746081Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T00:50:26.061285587Z caller=checkpoint.go:378 msg="writing series" size="269 kB"
level=debug ts=2026-04-21T00:50:26.061314341Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=11
level=info ts=2026-04-21T00:50:26.087373332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:50:26.095185486Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-21T00:50:27.30967199Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:50:29.20036631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:50:29.204048501Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:50:29.204074049Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:50:29.204097692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:50:29.206914772Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T00:50:29.810918755Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=debug ts=2026-04-21T00:50:29.957438979Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:50:32.309769354Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:50:33.560942865Z caller=checkpoint.go:378 msg="writing series" size="8.0 kB"
ts=2026-04-21T00:50:37.198642634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:50:37.31038037Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:50:37.310593565Z caller=checkpoint.go:378 msg="writing series" size="393 kB"
level=info ts=2026-04-21T00:50:38.582324236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:50:38.586884446Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-21T00:50:39.576893619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:50:39.582474561Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:50:39.877581931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:50:39.881385871Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T00:50:41.06070234Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=warn ts=2026-04-21T00:50:42.310448046Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:50:42.843148119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:50:42.847077237Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T00:50:43.519204179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:50:43.523451254Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T00:50:44.810892477Z caller=checkpoint.go:378 msg="writing series" size="192 kB"
level=warn ts=2026-04-21T00:50:47.310414678Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:50:47.425683493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:50:47.432557506Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T00:50:48.109979565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:50:48.113912851Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T00:50:48.56082342Z caller=checkpoint.go:378 msg="writing series" size="105 kB"
level=info ts=2026-04-21T00:50:49.335942809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:50:49.339475415Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T00:50:52.286197514Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:50:52.287251854Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-21T00:50:52.309584796Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:50:52.30958925Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:50:52.30961949Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:50:52.309651709Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:50:52.310696804Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-04-21T00:50:53.358657638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:50:53.363295819Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T00:50:53.833673697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:50:53.837205884Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-04-21T00:50:55.057415679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:43638"
level=info ts=2026-04-21T00:50:55.796085837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:50:55.79948785Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T00:50:56.057077385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T00:50:56.060238685Z caller=checkpoint.go:378 msg="writing series" size="613 kB"
level=debug ts=2026-04-21T00:50:56.060253929Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=8
level=error ts=2026-04-21T00:50:56.062106814Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-21T00:50:57.309746116Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:50:57.885431261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:50:57.890377876Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T00:50:59.689186585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:50:59.695980717Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:50:59.820050976Z caller=checkpoint.go:378 msg="writing series" size="13 MB"
level=debug ts=2026-04-21T00:50:59.820078752Z caller=checkpoint.go:390 msg="flushing series" totalSize="13 MB" series=1
level=debug ts=2026-04-21T00:51:00.59185393Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:51:02.31040965Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:51:03.500873194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:51:03.50463859Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T00:51:03.560697905Z caller=checkpoint.go:378 msg="writing series" size="657 B"
level=info ts=2026-04-21T00:51:06.579375905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:51:06.583353077Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-04-21T00:51:07.199815081Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:51:07.30960634Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:51:07.310939471Z caller=checkpoint.go:378 msg="writing series" size="563 kB"
level=info ts=2026-04-21T00:51:07.684296494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:51:07.688524014Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T00:51:09.074747113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:51:09.079171633Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T00:51:11.060697127Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=info ts=2026-04-21T00:51:11.148176962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:51:11.154491076Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-21T00:51:12.310328773Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:51:14.810530295Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=debug ts=2026-04-21T00:51:15.991068766Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T00:51:15.992719609Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T00:51:17.310258991Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:51:18.560833874Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=info ts=2026-04-21T00:51:19.181874269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:51:19.189361633Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T00:51:22.310396872Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:51:22.310430921Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:51:22.310439355Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:51:22.310460981Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:51:22.310551768Z caller=checkpoint.go:378 msg="writing series" size="175 kB"
level=info ts=2026-04-21T00:51:23.193853466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:51:23.197705222Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-21T00:51:26.060055678Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=warn ts=2026-04-21T00:51:27.310524754Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:51:27.532752381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:51:27.542369923Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T00:51:28.276165568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:51:28.279677457Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:51:28.279698403Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:51:28.279723882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:51:28.282951305Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-04-21T00:51:28.999857686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:55446"
level=info ts=2026-04-21T00:51:29.405749239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:51:29.409154819Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:51:29.409175175Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:51:29.409195947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:51:29.411956089Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T00:51:29.810237625Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=debug ts=2026-04-21T00:51:30.978903851Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:51:32.30966768Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:51:32.469330866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:51:32.473499674Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:51:33.560686871Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-04-21T00:51:35.90606142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:51:35.911297125Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:51:35.911319964Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:51:35.911344508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:51:35.914280287Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-04-21T00:51:37.201204845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:51:37.310509792Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:51:37.310536016Z caller=checkpoint.go:378 msg="writing series" size="817 B"
level=info ts=2026-04-21T00:51:38.612690512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:51:38.616311053Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:51:38.616332266Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:51:38.616356937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:51:38.619534109Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T00:51:41.060393785Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=info ts=2026-04-21T00:51:41.931201408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:51:41.936063221Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T00:51:41.964153344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:51:41.970851808Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T00:51:42.134551975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:51:42.138104923Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-04-21T00:51:42.138128914Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:51:42.138151038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:51:42.141245006Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-21T00:51:42.309576251Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:51:44.236295509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:51:44.242681434Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T00:51:44.810856697Z caller=checkpoint.go:378 msg="writing series" size="841 kB"
level=debug ts=2026-04-21T00:51:44.810883433Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=12
level=info ts=2026-04-21T00:51:45.683591197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:51:45.687240695Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:51:45.687261577Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:51:45.68728433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:51:45.69013435Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-04-21T00:51:47.31028917Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:51:48.118314998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:51:48.123574651Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T00:51:48.560251162Z caller=checkpoint.go:378 msg="writing series" size="219 kB"
level=info ts=2026-04-21T00:51:49.708384954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:51:49.713159755Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T00:51:50.479466746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:51:50.482980601Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:51:50.483005666Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:51:50.483029431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:51:50.486242145Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T00:51:51.295544083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:51:51.299130487Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T00:51:52.286515295Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:51:52.286923228Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:51:52.310353094Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:51:52.310378735Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:51:52.310387658Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:51:52.310379198Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:51:52.3106028Z caller=checkpoint.go:378 msg="writing series" size="445 kB"
level=info ts=2026-04-21T00:51:52.661364251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:51:52.665708455Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-21T00:51:53.155072912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:51:53.159103275Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T00:51:56.060117997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T00:51:56.060474659Z caller=checkpoint.go:378 msg="writing series" size="383 kB"
level=error ts=2026-04-21T00:51:56.064469303Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T00:51:56.226948317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:51:56.230527827Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T00:51:57.218556158Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:51:57.31056417Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:51:57.662185301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:51:57.667239889Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-04-21T00:51:58.929794814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:45668"
ts=2026-04-21T00:51:59.001811847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:37716"
level=info ts=2026-04-21T00:51:59.090572833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:51:59.094530216Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T00:51:59.810001163Z caller=checkpoint.go:378 msg="writing series" size="102 kB"
level=debug ts=2026-04-21T00:51:59.810031074Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-04-21T00:52:02.310472807Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:52:03.560346471Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=info ts=2026-04-21T00:52:03.68690137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:52:03.690814593Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-04-21T00:52:07.202937759Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:52:07.309789223Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:52:07.310870414Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-04-21T00:52:10.297124046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:52:10.300799588Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T00:52:10.726972529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:52:10.737429601Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T00:52:11.061054322Z caller=checkpoint.go:378 msg="writing series" size="435 kB"
level=info ts=2026-04-21T00:52:11.270596822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:52:11.274187758Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T00:52:11.7075117Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T00:52:12.310314268Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:52:13.931879372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:52:13.935902291Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-21T00:52:14.432554111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:52:14.436520283Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T00:52:14.811076738Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=info ts=2026-04-21T00:52:16.123527499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:52:16.130598972Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:52:16.130638209Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:52:16.130671857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:52:16.137111729Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T00:52:16.774853785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:52:16.778559649Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-21T00:52:17.30956554Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:52:17.422905908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:52:17.427005953Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:52:18.560040325Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-04-21T00:52:21.447964144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:52:21.452165499Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T00:52:22.095055084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:52:22.102464508Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T00:52:22.310115408Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:52:22.310145078Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-21T00:52:22.310132878Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:52:22.310153154Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:52:22.310178903Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=debug ts=2026-04-21T00:52:25.194370142Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:52:25.373336363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:52:25.377596334Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T00:52:26.060335155Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=warn ts=2026-04-21T00:52:27.310016738Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:52:29.810826583Z caller=checkpoint.go:378 msg="writing series" size="273 kB"
level=debug ts=2026-04-21T00:52:29.810855572Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-04-21T00:52:32.309986609Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:52:32.506358381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:52:32.513969145Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T00:52:33.267366462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:52:33.270874262Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T00:52:33.560014943Z caller=checkpoint.go:378 msg="writing series" size="8.2 kB"
level=info ts=2026-04-21T00:52:33.631291208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:52:33.634965131Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T00:52:34.46535112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:52:34.470715125Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-04-21T00:52:37.204793418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:52:37.310552717Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:52:37.310661315Z caller=checkpoint.go:378 msg="writing series" size="119 kB"
level=info ts=2026-04-21T00:52:38.518940661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:52:38.522787279Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-21T00:52:41.062911289Z caller=checkpoint.go:378 msg="writing series" size="2.5 MB"
level=debug ts=2026-04-21T00:52:41.062934481Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.7 MB" series=3
level=info ts=2026-04-21T00:52:41.435846676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:52:41.441202763Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-21T00:52:42.310089478Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:52:44.810314892Z caller=checkpoint.go:378 msg="writing series" size="601 kB"
level=warn ts=2026-04-21T00:52:47.309768425Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:52:48.210746966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:52:48.214566437Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T00:52:48.560867048Z caller=checkpoint.go:378 msg="writing series" size="869 kB"
level=debug ts=2026-04-21T00:52:48.56090101Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=2
level=info ts=2026-04-21T00:52:50.430385324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:52:50.434717891Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T00:52:51.408446178Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:52:52.286200924Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:52:52.287252174Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-21T00:52:52.309570408Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:52:52.309580239Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:52:52.309602492Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:52:52.309611535Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:52:52.310713692Z caller=checkpoint.go:378 msg="writing series" size="5.5 kB"
level=info ts=2026-04-21T00:52:53.863837599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:52:53.869214202Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-04-21T00:52:55.065345036Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:34720"
level=debug ts=2026-04-21T00:52:56.060543366Z caller=checkpoint.go:378 msg="writing series" size="336 kB"
level=warn ts=2026-04-21T00:52:57.310176877Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:52:58.777471294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:52:58.781214371Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T00:52:59.195806335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:52:59.199348329Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T00:52:59.811434251Z caller=checkpoint.go:378 msg="writing series" size="1.0 MB"
level=debug ts=2026-04-21T00:52:59.811457356Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=3
level=info ts=2026-04-21T00:53:00.296145448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:53:00.300096571Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-04-21T00:53:00.300114303Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:53:00.300138552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:53:00.303451969Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-21T00:53:01.462932363Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T00:53:01.636401687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:53:01.64058106Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-21T00:53:02.309612296Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:53:02.90962081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:53:02.918354142Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T00:53:03.560331808Z caller=checkpoint.go:378 msg="writing series" size="5.5 kB"
level=info ts=2026-04-21T00:53:04.055751768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:53:04.063884364Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T00:53:06.699743161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:53:06.703324772Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-04-21T00:53:07.206336163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:53:07.310370223Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:53:07.310588359Z caller=checkpoint.go:378 msg="writing series" size="437 kB"
level=info ts=2026-04-21T00:53:08.413061343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:53:08.416703302Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T00:53:11.060669141Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-04-21T00:53:11.715873849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:53:11.719708081Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-21T00:53:12.309842907Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:53:14.810694902Z caller=checkpoint.go:378 msg="writing series" size="195 kB"
level=warn ts=2026-04-21T00:53:17.310256499Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:53:18.560825172Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=info ts=2026-04-21T00:53:18.885977061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:53:18.892052568Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-21T00:53:18.961128493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:53:18.966296046Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T00:53:20.062684723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:53:20.070543757Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:53:20.541771279Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:53:21.69114621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:53:21.694727077Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T00:53:22.287117737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:53:22.290931923Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T00:53:22.310065433Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:53:22.310080624Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:53:22.310093347Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-04-21T00:53:22.310090573Z caller=checkpoint.go:390 msg="flushing series" totalSize="651 kB" series=5
level=info ts=2026-04-21T00:53:22.3101015Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T00:53:22.317443947Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000016.tmp new=/var/loki/wal/checkpoint.000016
level=info ts=2026-04-21T00:53:22.324237061Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.014400489s
ts=2026-04-21T00:53:25.066314979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:58326"
level=warn ts=2026-04-21T00:53:27.31017138Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:53:29.006921346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:39876"
level=info ts=2026-04-21T00:53:29.916846056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:53:29.920540006Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-21T00:53:31.585155479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:53:31.589960273Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-21T00:53:32.310118213Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:53:35.601921897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:53:35.606798093Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T00:53:36.432028111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:53:36.435941729Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-04-21T00:53:37.207850997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:53:37.309886868Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:53:41.801825857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:53:41.806221333Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T00:53:42.30950874Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:53:44.294496634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:53:44.29821787Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T00:53:45.752112114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:53:45.759791148Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T00:53:45.820895583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:53:45.824986502Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-04-21T00:53:47.310156613Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:53:52.075753251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:53:52.081199523Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T00:53:52.286394335Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:53:52.286410921Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:53:52.309754105Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-21T00:53:52.309764266Z caller=checkpoint.go:612 msg="starting checkpoint"
level=debug ts=2026-04-21T00:53:52.30980477Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:53:52.309813215Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:53:52.309863384Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:53:52.309927322Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000017
level=debug ts=2026-04-21T00:53:52.310109279Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=info ts=2026-04-21T00:53:53.084892306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:53:53.088724428Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T00:53:56.060881513Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=debug ts=2026-04-21T00:53:56.525563274Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:53:57.310243821Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:53:57.798909885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:53:57.804255871Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-04-21T00:53:58.936289922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:42016"
level=info ts=2026-04-21T00:53:58.966896025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:53:58.970877206Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-04-21T00:53:59.008311107Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:46892"
level=debug ts=2026-04-21T00:53:59.811155807Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=info ts=2026-04-21T00:54:01.16292189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:54:01.17037402Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T00:54:02.133005974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:54:02.140332206Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:54:02.140358048Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:54:02.140388858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:54:02.147175017Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T00:54:02.310091388Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:54:02.809742993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:54:02.813775834Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T00:54:03.345980754Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T00:54:03.560737377Z caller=checkpoint.go:378 msg="writing series" size="310 kB"
ts=2026-04-21T00:54:07.209557067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:54:07.31048544Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:54:07.310495733Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=info ts=2026-04-21T00:54:10.485412574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:54:10.489800162Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:54:10.489825654Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:54:10.489853136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:54:10.492978868Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-21T00:54:11.060803266Z caller=checkpoint.go:378 msg="writing series" size="689 kB"
level=debug ts=2026-04-21T00:54:11.060827482Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-04-21T00:54:11.627369549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:54:11.632805856Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-21T00:54:12.309711759Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:54:14.414975128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:54:14.421484677Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T00:54:14.810957128Z caller=checkpoint.go:378 msg="writing series" size="974 kB"
level=info ts=2026-04-21T00:54:16.169517993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:54:16.173253148Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-04-21T00:54:17.309869608Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:54:18.561949523Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-04-21T00:54:18.561975764Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.2 MB" series=2
level=info ts=2026-04-21T00:54:21.367434634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:54:21.371128495Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T00:54:21.523744119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:54:21.527334384Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T00:54:22.309427523Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:54:22.309459542Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:54:22.309470158Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:54:22.309497666Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:54:22.310724242Z caller=checkpoint.go:378 msg="writing series" size="218 kB"
level=info ts=2026-04-21T00:54:24.366603689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:54:24.371071858Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T00:54:26.061090489Z caller=checkpoint.go:378 msg="writing series" size="147 kB"
level=warn ts=2026-04-21T00:54:27.309901407Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:54:27.597817431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:54:27.602147697Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T00:54:28.634288195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:54:28.638341965Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-04-21T00:54:29.009803998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:33508"
level=debug ts=2026-04-21T00:54:29.521191047Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:54:29.730686156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:54:29.73598745Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-21T00:54:29.810210147Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=warn ts=2026-04-21T00:54:32.309650815Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:54:33.561202728Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=info ts=2026-04-21T00:54:34.72689219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:54:34.731152675Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-04-21T00:54:37.211424857Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:54:37.310204541Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:54:37.336363648Z caller=checkpoint.go:378 msg="writing series" size="23 MB"
level=debug ts=2026-04-21T00:54:37.336389975Z caller=checkpoint.go:390 msg="flushing series" totalSize="24 MB" series=5
level=info ts=2026-04-21T00:54:37.427550308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:54:37.43110118Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T00:54:37.698180891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:54:37.701759523Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T00:54:38.16102832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:54:38.165565962Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:54:38.546258326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:54:38.549987421Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T00:54:41.060411755Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=warn ts=2026-04-21T00:54:42.309515551Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:54:42.667501712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.8 MB" total_uncomp="183 MB" avg_uncomp="23 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:54:42.675330182Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:54:44.812053745Z caller=checkpoint.go:378 msg="writing series" size="1.5 MB"
level=debug ts=2026-04-21T00:54:44.812078775Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=2
level=warn ts=2026-04-21T00:54:47.310086291Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:54:48.560367125Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
level=debug ts=2026-04-21T00:54:49.669344556Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T00:54:49.74265995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:54:49.746165766Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-21T00:54:52.286186296Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:54:52.287254126Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T00:54:52.309553868Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:54:52.309567648Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:54:52.309587239Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:54:52.309598629Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:54:52.310690228Z caller=checkpoint.go:378 msg="writing series" size="7.2 kB"
level=info ts=2026-04-21T00:54:52.721559701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:54:52.72991425Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T00:54:56.06113771Z caller=checkpoint.go:378 msg="writing series" size="78 kB"
level=info ts=2026-04-21T00:54:56.658833108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:54:56.662517432Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T00:54:57.302288322Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:54:57.310342951Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:54:57.865955337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:54:57.873949937Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T00:54:57.977449306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:54:57.981185211Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T00:54:59.81113098Z caller=checkpoint.go:378 msg="writing series" size="280 kB"
level=info ts=2026-04-21T00:55:00.591829888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:55:00.598762746Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:55:00.598788847Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:55:00.598818793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:55:00.603603583Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T00:55:00.912441564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:55:00.916199938Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-21T00:55:02.310339849Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:55:03.560687545Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=info ts=2026-04-21T00:55:05.862140931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:55:05.865900106Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-04-21T00:55:07.212374619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:55:07.310036263Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:55:07.310203379Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=debug ts=2026-04-21T00:55:11.060816007Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=warn ts=2026-04-21T00:55:12.309853618Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:55:14.810501145Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-21T00:55:14.813600686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:55:14.818717237Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-21T00:55:14.984201811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:55:14.989684367Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:55:14.989710806Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:55:14.989740662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:55:14.994447361Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T00:55:15.091010618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:55:15.094539852Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-21T00:55:17.31042832Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:55:17.595938068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:55:17.600132732Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T00:55:18.560410211Z caller=checkpoint.go:378 msg="writing series" size="63 kB"
level=info ts=2026-04-21T00:55:18.586698387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:55:18.591513963Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T00:55:20.211326609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:55:20.214797441Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T00:55:20.739061399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:55:20.743156283Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:55:20.743178677Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:55:20.743206201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:55:20.746384982Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:55:22.309979507Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:55:22.309988643Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:55:22.310010623Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:55:22.310031694Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:55:22.311083323Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=debug ts=2026-04-21T00:55:23.348125945Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:55:24.197749104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:55:24.20124848Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-04-21T00:55:25.07406462Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:34134"
level=debug ts=2026-04-21T00:55:25.838755484Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T00:55:25.840282134Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-21T00:55:26.061293787Z caller=checkpoint.go:378 msg="writing series" size="269 kB"
level=info ts=2026-04-21T00:55:26.410850434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="23 MB" avg_comp="2.6 MB" total_uncomp="192 MB" avg_uncomp="21 MB" full=7 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:55:26.418536946Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T00:55:27.16767138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:55:27.171052454Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-21T00:55:27.310442906Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:55:27.846760018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:55:27.852154565Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-21T00:55:29.810440305Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=debug ts=2026-04-21T00:55:29.810466557Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=12
level=warn ts=2026-04-21T00:55:32.310319326Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:55:33.560478998Z caller=checkpoint.go:378 msg="writing series" size="8.0 kB"
level=info ts=2026-04-21T00:55:33.750097747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:55:33.754543443Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:55:34.281041534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:55:34.284619643Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-21T00:55:36.99923822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:55:37.002807489Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-04-21T00:55:37.21399384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:55:37.309998523Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:55:37.311117171Z caller=checkpoint.go:378 msg="writing series" size="213 kB"
level=info ts=2026-04-21T00:55:38.686607392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:55:38.693827988Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T00:55:41.060232695Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=info ts=2026-04-21T00:55:41.612133594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:55:41.620206572Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T00:55:42.155468432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:55:42.159281145Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-04-21T00:55:42.309855627Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:55:42.390174703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:55:42.394316371Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T00:55:44.811457459Z caller=checkpoint.go:378 msg="writing series" size="192 kB"
level=warn ts=2026-04-21T00:55:47.309763123Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:55:48.56120089Z caller=checkpoint.go:378 msg="writing series" size="105 kB"
level=debug ts=2026-04-21T00:55:52.28639433Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:55:52.286401077Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T00:55:52.309824024Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:55:52.309853352Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-21T00:55:52.30983982Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:55:52.30986184Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:55:52.310979357Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-04-21T00:55:53.245847907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:55:53.254205328Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T00:55:54.883756525Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:55:55.121926825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:55:55.129519677Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T00:55:56.060367337Z caller=checkpoint.go:378 msg="writing series" size="504 kB"
level=debug ts=2026-04-21T00:55:56.060392897Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-04-21T00:55:57.309889009Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:55:59.015551704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:53980"
level=debug ts=2026-04-21T00:55:59.818292104Z caller=checkpoint.go:378 msg="writing series" size="14 MB"
level=debug ts=2026-04-21T00:55:59.818318486Z caller=checkpoint.go:390 msg="flushing series" totalSize="14 MB" series=1
level=info ts=2026-04-21T00:56:00.195286804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:56:00.199532692Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T00:56:00.733978179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:56:00.740172931Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T00:56:01.318538727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:56:01.324246534Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-21T00:56:02.233835018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:56:02.237771546Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-21T00:56:02.309900586Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:56:03.48564738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:56:03.489880133Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T00:56:03.561000135Z caller=checkpoint.go:378 msg="writing series" size="657 B"
level=debug ts=2026-04-21T00:56:06.332754134Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T00:56:07.217257265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:56:07.310504101Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:56:07.310869435Z caller=checkpoint.go:378 msg="writing series" size="574 kB"
level=info ts=2026-04-21T00:56:07.546039467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:56:07.549496088Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T00:56:07.818295142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:56:07.82207723Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T00:56:11.00226455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="23 MB" avg_comp="2.6 MB" total_uncomp="192 MB" avg_uncomp="21 MB" full=7 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:56:11.009377379Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:56:11.060474888Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-21T00:56:12.310323852Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:56:14.810680552Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=info ts=2026-04-21T00:56:15.66219517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:56:15.666437503Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:56:15.666459994Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:56:15.666485189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:56:15.67006368Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T00:56:17.309688674Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:56:18.560155321Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=debug ts=2026-04-21T00:56:19.626403384Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:56:20.840479266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:56:20.844472603Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T00:56:22.309849274Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:56:22.309877988Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:56:22.309886341Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:56:22.309902581Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:56:22.311057181Z caller=checkpoint.go:378 msg="writing series" size="181 kB"
level=info ts=2026-04-21T00:56:23.528203214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:56:23.53568447Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:56:23.535709399Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:56:23.535738802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:56:23.541847834Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-21T00:56:25.077015037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:36342"
level=info ts=2026-04-21T00:56:25.160360567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:56:25.165856171Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-21T00:56:26.060897692Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=warn ts=2026-04-21T00:56:27.310178497Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:56:29.810620522Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=info ts=2026-04-21T00:56:29.962025887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:56:29.965903893Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T00:56:30.269665801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:56:30.272989352Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T00:56:30.506290043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:56:30.510415825Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T00:56:32.310102095Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:56:33.561258779Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-04-21T00:56:35.491187265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:56:35.498902584Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T00:56:35.945063596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:56:35.95178729Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-21T00:56:37.218553811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:56:37.309949549Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:56:37.310986944Z caller=checkpoint.go:378 msg="writing series" size="817 B"
level=info ts=2026-04-21T00:56:38.107135367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:56:38.112260817Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T00:56:41.060394027Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=info ts=2026-04-21T00:56:41.718506032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:56:41.722540188Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T00:56:41.739559391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:56:41.743641992Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-04-21T00:56:42.310063769Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:56:42.615517034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:56:42.61918126Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T00:56:44.811437973Z caller=checkpoint.go:378 msg="writing series" size="841 kB"
level=debug ts=2026-04-21T00:56:44.811461953Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=12
level=warn ts=2026-04-21T00:56:47.310422371Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:56:47.816900677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:56:47.820735768Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T00:56:48.560876842Z caller=checkpoint.go:378 msg="writing series" size="244 kB"
level=info ts=2026-04-21T00:56:50.444834631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:56:50.448393403Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T00:56:51.222602341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:56:51.226803884Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:56:51.612015282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:56:51.617152293Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T00:56:52.28639958Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:56:52.286655396Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:56:52.310076212Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:56:52.310108027Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:56:52.310118433Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:56:52.310125969Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:56:52.310367183Z caller=checkpoint.go:378 msg="writing series" size="524 kB"
level=debug ts=2026-04-21T00:56:54.197454581Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:56:56.061061292Z caller=checkpoint.go:378 msg="writing series" size="386 kB"
level=debug ts=2026-04-21T00:56:56.061086478Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=3
level=info ts=2026-04-21T00:56:56.07102252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:56:56.075421938Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:56:56.075439718Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:56:56.075459805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:56:56.078337852Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-21T00:56:57.31000494Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:56:59.019735809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:60168"
level=debug ts=2026-04-21T00:56:59.810739796Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=info ts=2026-04-21T00:57:01.829250452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="23 MB" avg_comp="2.6 MB" total_uncomp="192 MB" avg_uncomp="21 MB" full=7 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:57:01.836432433Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-21T00:57:02.30976406Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:57:02.897282805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:57:02.902002916Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T00:57:03.560953801Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=info ts=2026-04-21T00:57:04.218675779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:57:04.222686315Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:57:04.222710479Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:57:04.222734607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:57:04.22610966Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T00:57:06.61505339Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T00:57:07.220521832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:57:07.310409055Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:57:07.310435115Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-04-21T00:57:10.63574684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:57:10.639826866Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-21T00:57:11.061075343Z caller=checkpoint.go:378 msg="writing series" size="435 kB"
level=warn ts=2026-04-21T00:57:12.310360147Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:57:12.655096567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:57:12.658818085Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T00:57:14.462296118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:57:14.469204299Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:57:14.810395068Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=info ts=2026-04-21T00:57:15.224822105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:57:15.228202576Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-04-21T00:57:17.309850921Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:57:18.561065556Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-04-21T00:57:18.78384513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:57:18.787521961Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T00:57:19.775284952Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:57:20.691058098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:57:20.694837283Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T00:57:22.019835485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:57:22.023610962Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T00:57:22.310265603Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:57:22.31030168Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:57:22.310316332Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:57:22.310308371Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=warn ts=2026-04-21T00:57:22.310314475Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T00:57:25.080122218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:37326"
level=debug ts=2026-04-21T00:57:26.060609737Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=warn ts=2026-04-21T00:57:27.30955042Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:57:27.335679793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:57:27.343534529Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-21T00:57:28.949835419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:53556"
level=debug ts=2026-04-21T00:57:29.811073533Z caller=checkpoint.go:378 msg="writing series" size="319 kB"
level=debug ts=2026-04-21T00:57:29.811097688Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=9
level=info ts=2026-04-21T00:57:30.577181935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:57:30.580957061Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T00:57:31.73899542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:57:31.74414584Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-21T00:57:32.310063741Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:57:33.449567109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:57:33.453413042Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T00:57:33.560775875Z caller=checkpoint.go:378 msg="writing series" size="8.2 kB"
level=info ts=2026-04-21T00:57:34.584811848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:57:34.588793811Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T00:57:35.270667805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:57:35.27589309Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T00:57:35.285732345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:57:35.289980407Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-21T00:57:36.340789006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:57:36.344711163Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-04-21T00:57:37.222928502Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:57:37.310197181Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:57:37.310255307Z caller=checkpoint.go:378 msg="writing series" size="122 kB"
level=debug ts=2026-04-21T00:57:41.062250948Z caller=checkpoint.go:378 msg="writing series" size="2.6 MB"
level=debug ts=2026-04-21T00:57:41.062274829Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.7 MB" series=3
level=warn ts=2026-04-21T00:57:42.310339408Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:57:43.285158517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:57:43.288607128Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T00:57:43.288648198Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:57:43.288672146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="182 B" avg_comp="182 B" total_uncomp="19 kB" avg_uncomp="19 kB" idle=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-04-21T00:57:43.291644452Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19dad35a46e%3A19dad4d5459%3Acbe97912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-04-21T00:57:44.341007363Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T00:57:44.810656681Z caller=checkpoint.go:378 msg="writing series" size="476 kB"
level=debug ts=2026-04-21T00:57:46.457333773Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:57:47.309747317Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:57:47.522034313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:57:47.525698541Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T00:57:48.560740329Z caller=checkpoint.go:378 msg="writing series" size="869 kB"
level=debug ts=2026-04-21T00:57:48.560767665Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=2
level=info ts=2026-04-21T00:57:49.404841422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:57:49.40878729Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:57:51.65260626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:57:51.656197734Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-21T00:57:52.286282911Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:57:52.287331232Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-21T00:57:52.309679459Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:57:52.309667151Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:57:52.309702752Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:57:52.309715522Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:57:52.310828296Z caller=checkpoint.go:378 msg="writing series" size="5.7 kB"
level=debug ts=2026-04-21T00:57:56.06060445Z caller=checkpoint.go:378 msg="writing series" size="336 kB"
level=info ts=2026-04-21T00:57:56.228036049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:57:56.232283076Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T00:57:57.310445029Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:57:57.53793897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-04-21T00:57:57.544893122Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19dad35a46e%3A19dad4d5459%3Acbe97912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-04-21T00:57:58.932636321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="23 MB" avg_comp="2.6 MB" total_uncomp="192 MB" avg_uncomp="21 MB" full=7 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:57:58.939753618Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:57:59.811033939Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-21T00:57:59.811058331Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=3
level=info ts=2026-04-21T00:58:01.277503265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:58:01.281404954Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:58:01.281428278Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:58:01.281452171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:58:01.284460456Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-04-21T00:58:02.310367157Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:58:03.560417925Z caller=checkpoint.go:378 msg="writing series" size="5.5 kB"
ts=2026-04-21T00:58:07.225015443Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T00:58:07.309979013Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:58:07.311233426Z caller=checkpoint.go:378 msg="writing series" size="437 kB"
level=info ts=2026-04-21T00:58:07.425314787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:58:07.430405581Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:58:07.430431108Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:58:07.430456311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:58:07.433519212Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T00:58:08.290057287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:58:08.296389452Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T00:58:11.060938904Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-04-21T00:58:11.188549123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:58:11.195176209Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T00:58:11.638364403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:58:11.64176529Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-04-21T00:58:12.309572838Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:58:12.359742348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:58:12.363260143Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:58:12.363279823Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:58:12.363299974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:58:12.366561314Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T00:58:14.548607986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:58:14.552168941Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:58:14.552192097Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:58:14.55221391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:58:14.55515971Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T00:58:14.811149052Z caller=checkpoint.go:378 msg="writing series" size="196 kB"
level=warn ts=2026-04-21T00:58:17.31037647Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:58:17.631950058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-04-21T00:58:17.635778049Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19dad35a46e%3A19dad4d5459%3Acbe97912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-04-21T00:58:17.793735779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:58:17.797338887Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T00:58:18.560698191Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=debug ts=2026-04-21T00:58:20.219145983Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T00:58:22.309597965Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:58:22.309636474Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:58:22.309663576Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:58:22.30967438Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:58:22.310741698Z caller=checkpoint.go:390 msg="flushing series" totalSize="652 kB" series=5
level=info ts=2026-04-21T00:58:22.31644445Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000017.tmp new=/var/loki/wal/checkpoint.000017
level=info ts=2026-04-21T00:58:22.320571385Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.010501914s
level=info ts=2026-04-21T00:58:23.327237078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:58:23.332107713Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-21T00:58:23.343304717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:58:23.346771181Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-04-21T00:58:23.346792276Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:58:23.346814376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:58:23.349750558Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-04-21T00:58:25.083273757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:51938"
level=info ts=2026-04-21T00:58:25.624245242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:58:25.630035951Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T00:58:27.247310141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:58:27.251482916Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-21T00:58:27.310277532Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:58:27.372434666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:58:27.375751997Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-04-21T00:58:28.953439145Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:41368"
level=info ts=2026-04-21T00:58:28.960024704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:58:28.964239286Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T00:58:29.909183233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:58:29.912655327Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T00:58:31.175644479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:58:31.179301319Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:58:31.179324499Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:58:31.17934704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:58:31.182544852Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-21T00:58:32.309691953Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:58:35.005658139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:58:35.010868953Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-04-21T00:58:37.22748711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:58:37.310529085Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:58:38.760579806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:58:38.764784872Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T00:58:41.013314636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:58:41.017805512Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-21T00:58:41.610364639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:58:41.614269718Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-21T00:58:42.310140516Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:58:42.355231442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:58:42.359771229Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T00:58:42.91761434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:58:42.921098604Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T00:58:45.481282722Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T00:58:47.310406757Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:58:48.807564592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:58:48.811160274Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T00:58:49.287585324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:58:49.291549482Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:58:52.286386568Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:58:52.286793787Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:58:52.310194854Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-21T00:58:52.310203752Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-04-21T00:58:52.310206112Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:58:52.310227935Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:58:52.310237018Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T00:58:52.31048796Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000018
level=debug ts=2026-04-21T00:58:52.310700106Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=debug ts=2026-04-21T00:58:53.640843882Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:58:53.64090739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9b2c21dd0f9b9a09 immediate=false num_chunks=1 total_comp="608 kB" avg_comp="608 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" idle=1 labels="{stream=\"build-e2e-lzeq\"}"
level=error ts=2026-04-21T00:58:53.653901163Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9b2c21dd0f9b9a09/19dad4476d4%3A19dad54bae0%3A7765f6b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lzeq\"}"
level=info ts=2026-04-21T00:58:54.699901136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:58:54.703382028Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T00:58:56.061754152Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-04-21T00:58:56.68289659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="23 MB" avg_comp="2.6 MB" total_uncomp="192 MB" avg_uncomp="21 MB" full=7 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:58:56.689898801Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:58:56.689922426Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:58:56.689957236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="23 MB" avg_comp="2.6 MB" total_uncomp="192 MB" avg_uncomp="21 MB" full=7 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:58:56.696646399Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-21T00:58:57.310073828Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:58:59.679837511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:58:59.688149855Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T00:58:59.811353717Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=info ts=2026-04-21T00:59:00.476914255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:59:00.483449348Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T00:59:02.31050534Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:59:02.555964898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:59:02.559452399Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T00:59:03.561099101Z caller=checkpoint.go:378 msg="writing series" size="122 kB"
level=info ts=2026-04-21T00:59:04.722441424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:59:04.726436006Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-04-21T00:59:07.229303673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T00:59:07.309910863Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:59:07.311047899Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=info ts=2026-04-21T00:59:07.459329093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:59:07.465022704Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T00:59:08.195126479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-04-21T00:59:08.198893173Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19dad35a46e%3A19dad4d5459%3Acbe97912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-04-21T00:59:10.014840043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9b2c21dd0f9b9a09 immediate=false num_chunks=1 total_comp="642 kB" avg_comp="642 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" full=1 labels="{stream=\"build-e2e-lzeq\"}"
level=error ts=2026-04-21T00:59:10.020559878Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9b2c21dd0f9b9a09/19dad4476d4%3A19dad54bae0%3A7765f6b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lzeq\"}"
level=info ts=2026-04-21T00:59:10.759253302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="23 MB" avg_comp="2.6 MB" total_uncomp="192 MB" avg_uncomp="21 MB" full=7 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:59:10.766169126Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T00:59:11.061653317Z caller=checkpoint.go:378 msg="writing series" size="689 kB"
level=warn ts=2026-04-21T00:59:12.309944702Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:59:14.261105329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T00:59:14.264672093Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T00:59:14.408350292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:59:14.412316684Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-04-21T00:59:14.41234197Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T00:59:14.412369085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:59:14.415965233Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-21T00:59:14.811478377Z caller=checkpoint.go:378 msg="writing series" size="975 kB"
level=debug ts=2026-04-21T00:59:14.811503649Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=7
level=warn ts=2026-04-21T00:59:17.309763127Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:59:18.561945099Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-21T00:59:18.561975934Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=1
level=info ts=2026-04-21T00:59:19.279351219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T00:59:19.284440811Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T00:59:22.310238901Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T00:59:22.310240073Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:59:22.310266539Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:59:22.310275947Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T00:59:22.311641979Z caller=checkpoint.go:378 msg="writing series" size="221 kB"
level=info ts=2026-04-21T00:59:23.845327786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T00:59:23.848940899Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-21T00:59:24.243046412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T00:59:24.248670488Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T00:59:24.328180857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T00:59:24.332014646Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T00:59:25.289931253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T00:59:25.294056049Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T00:59:25.446451056Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T00:59:26.061991077Z caller=checkpoint.go:378 msg="writing series" size="158 kB"
level=warn ts=2026-04-21T00:59:27.309822451Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:59:27.397122688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:59:27.401358247Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-21T00:59:28.048657965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T00:59:28.052552805Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T00:59:28.117050652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T00:59:28.120338044Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T00:59:29.811330123Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=warn ts=2026-04-21T00:59:32.309511553Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:59:33.56148716Z caller=checkpoint.go:378 msg="writing series" size="191 kB"
ts=2026-04-21T00:59:37.230505411Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T00:59:37.309686505Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:59:37.333264124Z caller=checkpoint.go:378 msg="writing series" size="23 MB"
level=debug ts=2026-04-21T00:59:37.333292861Z caller=checkpoint.go:390 msg="flushing series" totalSize="24 MB" series=5
level=debug ts=2026-04-21T00:59:38.938693508Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T00:59:38.940386853Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-21T00:59:41.061572952Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=info ts=2026-04-21T00:59:41.523308723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T00:59:41.529381324Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-21T00:59:42.310518658Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:59:44.811679941Z caller=checkpoint.go:378 msg="writing series" size="1.6 MB"
level=debug ts=2026-04-21T00:59:44.811705792Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=2
level=info ts=2026-04-21T00:59:45.631014999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9b2c21dd0f9b9a09 immediate=false num_chunks=1 total_comp="642 kB" avg_comp="642 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" full=1 labels="{stream=\"build-e2e-lzeq\"}"
level=error ts=2026-04-21T00:59:45.635968965Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9b2c21dd0f9b9a09/19dad4476d4%3A19dad54bae0%3A7765f6b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lzeq\"}"
level=info ts=2026-04-21T00:59:45.979133604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T00:59:45.983791327Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T00:59:47.278465315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T00:59:47.285323184Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T00:59:47.310540762Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:59:48.561872101Z caller=checkpoint.go:378 msg="writing series" size="264 kB"
level=info ts=2026-04-21T00:59:48.728394187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="23 MB" avg_comp="2.6 MB" total_uncomp="192 MB" avg_uncomp="21 MB" full=7 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T00:59:48.736285445Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T00:59:49.506090211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T00:59:49.510150542Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T00:59:52.286416836Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T00:59:52.286450566Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T00:59:52.309807405Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T00:59:52.309840962Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T00:59:52.309851367Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T00:59:52.30992375Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T00:59:52.310961372Z caller=checkpoint.go:378 msg="writing series" size="7.2 kB"
level=debug ts=2026-04-21T00:59:52.725684517Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T00:59:55.019177011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T00:59:55.02304651Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T00:59:55.737076564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-04-21T00:59:55.741118178Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19dad35a46e%3A19dad4d5459%3Acbe97912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-04-21T00:59:56.060960592Z caller=checkpoint.go:378 msg="writing series" size="78 kB"
level=warn ts=2026-04-21T00:59:57.310377161Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T00:59:57.844948067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T00:59:57.850218579Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T00:59:58.205834417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T00:59:58.212549913Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T00:59:58.722900494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T00:59:58.727097222Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-21T00:59:59.810993743Z caller=checkpoint.go:378 msg="writing series" size="280 kB"
level=info ts=2026-04-21T01:00:01.947403438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T01:00:01.951099923Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-21T01:00:02.310265788Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:00:03.561108219Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=info ts=2026-04-21T01:00:06.4098076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T01:00:06.418066908Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-04-21T01:00:07.232124665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T01:00:07.309739878Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:00:07.310869454Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=debug ts=2026-04-21T01:00:11.060915836Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
level=warn ts=2026-04-21T01:00:12.309838324Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:00:12.669927209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T01:00:12.675100322Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T01:00:14.811232343Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-21T01:00:17.146514746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T01:00:17.151075015Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T01:00:17.310490162Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:00:18.561563494Z caller=checkpoint.go:378 msg="writing series" size="78 kB"
level=info ts=2026-04-21T01:00:19.540539927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T01:00:19.545814298Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T01:00:20.055996383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T01:00:20.059824152Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T01:00:20.05985677Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T01:00:20.059888871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T01:00:20.063074892Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-21T01:00:20.880913234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T01:00:20.886086703Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T01:00:20.894136989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T01:00:20.89721034Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T01:00:22.088232156Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T01:00:22.310209141Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T01:00:22.310223731Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:00:22.310241337Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T01:00:22.310249534Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T01:00:22.311361046Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=debug ts=2026-04-21T01:00:25.468056252Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T01:00:26.050603057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9b2c21dd0f9b9a09 immediate=false num_chunks=1 total_comp="642 kB" avg_comp="642 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" full=1 labels="{stream=\"build-e2e-lzeq\"}"
level=error ts=2026-04-21T01:00:26.054994518Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9b2c21dd0f9b9a09/19dad4476d4%3A19dad54bae0%3A7765f6b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lzeq\"}"
level=debug ts=2026-04-21T01:00:26.061197917Z caller=checkpoint.go:378 msg="writing series" size="269 kB"
level=debug ts=2026-04-21T01:00:26.061220627Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-04-21T01:00:27.309743146Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:00:28.254852831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T01:00:28.258918721Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-04-21T01:00:29.032983886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:50566"
level=debug ts=2026-04-21T01:00:29.810907167Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=info ts=2026-04-21T01:00:29.942752169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T01:00:29.949175067Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T01:00:29.949198512Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T01:00:29.949226133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T01:00:29.955040947Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T01:00:32.310212448Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:00:32.511558798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T01:00:32.515177549Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-21T01:00:33.561724492Z caller=checkpoint.go:378 msg="writing series" size="8.7 kB"
level=info ts=2026-04-21T01:00:34.065079591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T01:00:34.068601825Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-04-21T01:00:37.233362463Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T01:00:37.310158725Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:00:37.311412582Z caller=checkpoint.go:378 msg="writing series" size="282 kB"
level=info ts=2026-04-21T01:00:39.724419933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T01:00:39.729221065Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T01:00:41.061324607Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=info ts=2026-04-21T01:00:41.948913943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="23 MB" avg_comp="2.6 MB" total_uncomp="192 MB" avg_uncomp="21 MB" full=7 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T01:00:41.956056428Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-21T01:00:42.309987635Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:00:42.423260844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-04-21T01:00:42.426798497Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19dad35a46e%3A19dad4d5459%3Acbe97912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-04-21T01:00:44.811755328Z caller=checkpoint.go:378 msg="writing series" size="192 kB"
level=warn ts=2026-04-21T01:00:47.310210991Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:00:48.561129725Z caller=checkpoint.go:378 msg="writing series" size="106 kB"
level=info ts=2026-04-21T01:00:48.690597528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T01:00:48.697137675Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T01:00:51.351433773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T01:00:51.356924525Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T01:00:52.103278307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T01:00:52.107271955Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-21T01:00:52.286506639Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T01:00:52.286527357Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=debug ts=2026-04-21T01:00:52.287804851Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-21T01:00:52.287847791Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974147 n=1
level=debug ts=2026-04-21T01:00:52.287890013Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974147 n=1
level=info ts=2026-04-21T01:00:52.309940009Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T01:00:52.309972945Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T01:00:52.309981371Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T01:00:52.309975928Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:00:52.311097304Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-04-21T01:00:52.924413752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T01:00:52.928167772Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-04-21T01:00:55.091351177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:57492"
level=debug ts=2026-04-21T01:00:56.062007519Z caller=checkpoint.go:378 msg="writing series" size="618 kB"
level=debug ts=2026-04-21T01:00:56.0620336Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=8
level=info ts=2026-04-21T01:00:56.488260815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T01:00:56.492186923Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T01:00:57.164916403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T01:00:57.168704519Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-04-21T01:00:57.31027584Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:00:57.476699315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T01:00:57.483610663Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T01:00:58.028029635Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T01:00:58.053075979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T01:00:58.057470142Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-21T01:00:59.035333223Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:56658"
level=debug ts=2026-04-21T01:00:59.818560357Z caller=checkpoint.go:378 msg="writing series" size="14 MB"
level=debug ts=2026-04-21T01:00:59.818587094Z caller=checkpoint.go:390 msg="flushing series" totalSize="14 MB" series=1
level=info ts=2026-04-21T01:01:01.006166925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T01:01:01.009952701Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-21T01:01:02.31029414Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:01:03.561703582Z caller=checkpoint.go:378 msg="writing series" size="657 B"
level=info ts=2026-04-21T01:01:04.427259447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T01:01:04.430826342Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T01:01:06.33602185Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T01:01:07.069083273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T01:01:07.072527245Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-21T01:01:07.171922444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T01:01:07.176834194Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-04-21T01:01:07.235534761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T01:01:07.309526265Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:01:07.311956284Z caller=checkpoint.go:378 msg="writing series" size="599 kB"
level=info ts=2026-04-21T01:01:10.965016256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T01:01:10.970942737Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T01:01:11.061222348Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=info ts=2026-04-21T01:01:11.543836967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9b2c21dd0f9b9a09 immediate=false num_chunks=1 total_comp="642 kB" avg_comp="642 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" full=1 labels="{stream=\"build-e2e-lzeq\"}"
level=error ts=2026-04-21T01:01:11.548357578Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9b2c21dd0f9b9a09/19dad4476d4%3A19dad54bae0%3A7765f6b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lzeq\"}"
level=warn ts=2026-04-21T01:01:12.310155983Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:01:13.143598801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T01:01:13.147287647Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T01:01:13.631919848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T01:01:13.638361904Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T01:01:14.810930044Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=warn ts=2026-04-21T01:01:17.309734724Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:01:18.561687358Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=info ts=2026-04-21T01:01:21.963065466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="23 MB" avg_comp="2.6 MB" total_uncomp="192 MB" avg_uncomp="21 MB" full=7 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T01:01:21.971528618Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T01:01:22.309554783Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T01:01:22.309595585Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T01:01:22.309607369Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T01:01:22.309602952Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:01:22.31184413Z caller=checkpoint.go:378 msg="writing series" size="188 kB"
level=info ts=2026-04-21T01:01:25.737432644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T01:01:25.741742196Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T01:01:26.061117138Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=warn ts=2026-04-21T01:01:27.309993747Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:01:27.819781395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T01:01:27.824144068Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T01:01:29.810983055Z caller=checkpoint.go:378 msg="writing series" size="55 kB"
level=debug ts=2026-04-21T01:01:30.517150856Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T01:01:32.310423968Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:01:33.561291944Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-04-21T01:01:34.10796644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T01:01:34.11356593Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T01:01:34.113591308Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T01:01:34.113618535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T01:01:34.11892964Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-04-21T01:01:37.236986535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=warn ts=2026-04-21T01:01:37.309970756Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:01:37.311031851Z caller=checkpoint.go:378 msg="writing series" size="817 B"
level=info ts=2026-04-21T01:01:38.74964829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-04-21T01:01:38.75370876Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19dad35a46e%3A19dad4d5459%3Acbe97912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-04-21T01:01:39.131754222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="126 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T01:01:39.139858343Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T01:01:39.760377163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T01:01:39.764372887Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T01:01:40.082768084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T01:01:40.086263924Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T01:01:41.061462666Z caller=checkpoint.go:378 msg="writing series" size="5.2 kB"
level=warn ts=2026-04-21T01:01:42.31031244Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:01:42.79668261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T01:01:42.801431412Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-21T01:01:43.589646689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T01:01:43.594273624Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T01:01:43.761759028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T01:01:43.765279454Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T01:01:44.812181111Z caller=checkpoint.go:378 msg="writing series" size="642 kB"
level=debug ts=2026-04-21T01:01:44.812208404Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=12
level=info ts=2026-04-21T01:01:46.020831749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T01:01:46.024586789Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-21T01:01:47.310109025Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:01:47.512520457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T01:01:47.516757467Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-21T01:01:48.561890062Z caller=checkpoint.go:378 msg="writing series" size="270 kB"
level=info ts=2026-04-21T01:01:50.361673678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T01:01:50.3669925Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T01:01:52.286477518Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T01:01:52.286494371Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T01:01:52.309954936Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T01:01:52.309988051Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T01:01:52.309998007Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T01:01:52.309992051Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:01:52.311350018Z caller=checkpoint.go:378 msg="writing series" size="552 kB"
level=info ts=2026-04-21T01:01:54.147620947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T01:01:54.152128075Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T01:01:55.012907366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T01:01:55.016461968Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-04-21T01:01:55.094572294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:49696"
level=debug ts=2026-04-21T01:01:56.061293235Z caller=checkpoint.go:378 msg="writing series" size="436 kB"
level=debug ts=2026-04-21T01:01:56.061325894Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=3
level=warn ts=2026-04-21T01:01:57.309997545Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:01:57.685099067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T01:01:57.691489407Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T01:01:59.811811465Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
level=info ts=2026-04-21T01:02:00.468919639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9b2c21dd0f9b9a09 immediate=false num_chunks=1 total_comp="642 kB" avg_comp="642 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" full=1 labels="{stream=\"build-e2e-lzeq\"}"
level=error ts=2026-04-21T01:02:00.473886795Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9b2c21dd0f9b9a09/19dad4476d4%3A19dad54bae0%3A7765f6b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lzeq\"}"
level=warn ts=2026-04-21T01:02:02.309992497Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:02:02.977886965Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T01:02:03.561703805Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=info ts=2026-04-21T01:02:03.674978213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T01:02:03.679184286Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-04-21T01:02:03.679207729Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T01:02:03.679236994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T01:02:03.684386211Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T01:02:03.848679193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="23 MB" avg_comp="2.6 MB" total_uncomp="192 MB" avg_uncomp="21 MB" full=7 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T01:02:03.855770136Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T01:02:05.009684353Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T01:02:06.413809599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T01:02:06.418875163Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T01:02:06.418901623Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T01:02:06.418929781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T01:02:06.423353689Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-04-21T01:02:07.239319935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T01:02:07.310590605Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:02:07.311648122Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=debug ts=2026-04-21T01:02:11.06158914Z caller=checkpoint.go:378 msg="writing series" size="443 kB"
level=warn ts=2026-04-21T01:02:12.310218198Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:02:14.257266432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T01:02:14.263009951Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T01:02:14.4687611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T01:02:14.472379093Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T01:02:14.811482009Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=warn ts=2026-04-21T01:02:17.31026249Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:02:18.18547095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T01:02:18.190656167Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T01:02:18.561683552Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-04-21T01:02:21.348756524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T01:02:21.352435892Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T01:02:22.309937142Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T01:02:22.30996756Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T01:02:22.309976053Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T01:02:22.310025374Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:02:22.311068323Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-04-21T01:02:22.719169602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T01:02:22.72329577Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T01:02:23.76621914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="126 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T01:02:23.772751399Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T01:02:26.0611756Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=warn ts=2026-04-21T01:02:27.309783606Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T01:02:28.965445835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:49274"
level=debug ts=2026-04-21T01:02:29.442205483Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T01:02:29.8114033Z caller=checkpoint.go:378 msg="writing series" size="360 kB"
level=debug ts=2026-04-21T01:02:29.811429345Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=9
level=info ts=2026-04-21T01:02:30.360229035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T01:02:30.365045262Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T01:02:30.834368223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-04-21T01:02:30.838470912Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19dad35a46e%3A19dad4d5459%3Acbe97912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-04-21T01:02:31.042049646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T01:02:31.046150029Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-21T01:02:31.860294515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T01:02:31.864458632Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-21T01:02:32.309685965Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:02:33.561298231Z caller=checkpoint.go:378 msg="writing series" size="8.2 kB"
level=info ts=2026-04-21T01:02:33.753995371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T01:02:33.75805527Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-21T01:02:33.758082561Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T01:02:33.758112477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T01:02:33.761813767Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T01:02:33.916386249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T01:02:33.919893848Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T01:02:36.11741292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T01:02:36.121242568Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T01:02:36.664640637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T01:02:36.668159778Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-04-21T01:02:37.241435802Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T01:02:37.310212859Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:02:37.311453279Z caller=checkpoint.go:378 msg="writing series" size="125 kB"
level=info ts=2026-04-21T01:02:40.71418605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9b2c21dd0f9b9a09 immediate=false num_chunks=1 total_comp="642 kB" avg_comp="642 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" full=1 labels="{stream=\"build-e2e-lzeq\"}"
level=error ts=2026-04-21T01:02:40.718554531Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9b2c21dd0f9b9a09/19dad4476d4%3A19dad54bae0%3A7765f6b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lzeq\"}"
level=debug ts=2026-04-21T01:02:41.062818916Z caller=checkpoint.go:378 msg="writing series" size="2.5 MB"
level=debug ts=2026-04-21T01:02:41.06284316Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.6 MB" series=3
level=info ts=2026-04-21T01:02:41.983354292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T01:02:41.987765754Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-04-21T01:02:42.309753463Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:02:43.730909464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T01:02:43.734837614Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T01:02:44.811449992Z caller=checkpoint.go:378 msg="writing series" size="615 kB"
level=info ts=2026-04-21T01:02:46.070847672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="23 MB" avg_comp="2.6 MB" total_uncomp="192 MB" avg_uncomp="21 MB" full=7 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T01:02:46.078370595Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-21T01:02:47.310374622Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:02:48.561885638Z caller=checkpoint.go:378 msg="writing series" size="869 kB"
level=debug ts=2026-04-21T01:02:48.561910878Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=2
level=info ts=2026-04-21T01:02:50.758885687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T01:02:50.7633899Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T01:02:51.409306228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T01:02:51.412778735Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T01:02:52.286399617Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T01:02:52.287178026Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T01:02:52.309431732Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T01:02:52.30946214Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T01:02:52.309472298Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T01:02:52.309506281Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:02:52.311684464Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=debug ts=2026-04-21T01:02:56.060936932Z caller=checkpoint.go:378 msg="writing series" size="336 kB"
level=info ts=2026-04-21T01:02:56.473999503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T01:02:56.480297267Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T01:02:57.310440478Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:02:58.185642793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T01:02:58.190615079Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-04-21T01:02:59.042805447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:57500"
level=debug ts=2026-04-21T01:02:59.811465251Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-21T01:02:59.811489255Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=3
level=debug ts=2026-04-21T01:03:00.887968127Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T01:03:00.92717175Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T01:03:00.927247771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="668 B" avg_comp="668 B" total_uncomp="53 kB" avg_uncomp="53 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T01:03:00.931323131Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dad35c515%3A19dad58d1c0%3Acc49151d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-04-21T01:03:02.309538253Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:03:03.561559068Z caller=checkpoint.go:378 msg="writing series" size="5.5 kB"
ts=2026-04-21T01:03:07.243786272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=info ts=2026-04-21T01:03:07.245597213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T01:03:07.249243216Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-21T01:03:07.310224044Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:03:07.311492332Z caller=checkpoint.go:378 msg="writing series" size="437 kB"
level=info ts=2026-04-21T01:03:07.802214726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T01:03:07.807048344Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T01:03:07.91191789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T01:03:07.93402265Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T01:03:11.060897827Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-21T01:03:12.310502879Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:03:14.70540841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-04-21T01:03:14.709430003Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19dad35a46e%3A19dad4d5459%3Acbe97912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-04-21T01:03:14.811803195Z caller=checkpoint.go:378 msg="writing series" size="196 kB"
level=info ts=2026-04-21T01:03:15.105126453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T01:03:15.108686253Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-21T01:03:17.31025607Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:03:17.319418235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="8.2 kB" avg_comp="8.2 kB" total_uncomp="53 kB" avg_uncomp="53 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T01:03:17.323536834Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dad35c515%3A19dad58d1c0%3Acc49151d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-21T01:03:17.982821236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T01:03:17.986298922Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T01:03:18.561192511Z caller=checkpoint.go:378 msg="writing series" size="995 B"
level=info ts=2026-04-21T01:03:20.102016637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="126 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T01:03:20.108959152Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T01:03:20.108984023Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T01:03:20.109013671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="126 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T01:03:20.115522636Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T01:03:22.309780353Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T01:03:22.309810711Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T01:03:22.309822809Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T01:03:22.309836996Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:03:22.310896274Z caller=checkpoint.go:390 msg="flushing series" totalSize="653 kB" series=5
level=info ts=2026-04-21T01:03:22.31839111Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000018.tmp new=/var/loki/wal/checkpoint.000018
level=info ts=2026-04-21T01:03:22.321026444Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.010374039s
level=info ts=2026-04-21T01:03:24.310442105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T01:03:24.314208118Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T01:03:24.314232423Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T01:03:24.31426012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T01:03:24.317650484Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T01:03:24.823833548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T01:03:24.827961555Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-21T01:03:25.100511399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:39752"
level=info ts=2026-04-21T01:03:25.936236492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T01:03:25.940776585Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-21T01:03:26.981035541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T01:03:26.98454907Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-04-21T01:03:27.310175182Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:03:30.85545736Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T01:03:32.309681131Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:03:33.478162854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T01:03:33.483180433Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-21T01:03:35.430680372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T01:03:35.434305805Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T01:03:35.760015807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9b2c21dd0f9b9a09 immediate=false num_chunks=1 total_comp="642 kB" avg_comp="642 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" full=1 labels="{stream=\"build-e2e-lzeq\"}"
level=error ts=2026-04-21T01:03:35.76475867Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9b2c21dd0f9b9a09/19dad4476d4%3A19dad54bae0%3A7765f6b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lzeq\"}"
level=info ts=2026-04-21T01:03:36.450643148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T01:03:36.454230266Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-04-21T01:03:36.454251704Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T01:03:36.454272132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T01:03:36.457481847Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T01:03:37.244858091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="126 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
ts=2026-04-21T01:03:37.245538693Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=error ts=2026-04-21T01:03:37.251333188Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T01:03:37.309681319Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:03:37.365849432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="8.2 kB" avg_comp="8.2 kB" total_uncomp="53 kB" avg_uncomp="53 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T01:03:37.369532906Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dad35c515%3A19dad58d1c0%3Acc49151d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-21T01:03:37.404808068Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T01:03:37.406333092Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T01:03:37.47847114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T01:03:37.482294629Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T01:03:40.98488171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="23 MB" avg_comp="2.6 MB" total_uncomp="192 MB" avg_uncomp="21 MB" full=7 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T01:03:40.992000031Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T01:03:41.093413703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T01:03:41.099585064Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T01:03:42.310327501Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:03:43.880315936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T01:03:43.884192167Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-04-21T01:03:47.31050243Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:03:50.805533582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T01:03:50.811988055Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T01:03:51.900190694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T01:03:51.904057149Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T01:03:52.286641559Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T01:03:52.286924152Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T01:03:52.309289457Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-21T01:03:52.309489702Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000019
level=warn ts=2026-04-21T01:03:52.309521961Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:03:52.309518749Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T01:03:52.30954203Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T01:03:52.309551086Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T01:03:52.309761884Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=info ts=2026-04-21T01:03:55.32216213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T01:03:55.326092306Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T01:03:56.060655168Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=warn ts=2026-04-21T01:03:57.310137177Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:03:58.40643408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="126 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T01:03:58.414396897Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T01:03:58.840714938Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T01:03:59.810552185Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=info ts=2026-04-21T01:04:01.023064505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T01:04:01.028520375Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-21T01:04:02.309836579Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:04:03.560611085Z caller=checkpoint.go:378 msg="writing series" size="274 kB"
level=info ts=2026-04-21T01:04:05.011227642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T01:04:05.016228705Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-04-21T01:04:07.247993091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T01:04:07.309880142Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:04:07.309935027Z caller=checkpoint.go:378 msg="writing series" size="62 kB"
level=info ts=2026-04-21T01:04:08.997177398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T01:04:09.001517017Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T01:04:09.757478683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T01:04:09.761029707Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T01:04:09.76105191Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T01:04:09.761077241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T01:04:09.764177316Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T01:04:10.735195292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T01:04:10.739033055Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T01:04:11.060022619Z caller=checkpoint.go:378 msg="writing series" size="528 kB"
level=warn ts=2026-04-21T01:04:12.310260589Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:04:13.205145826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T01:04:13.20876229Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T01:04:13.384534567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T01:04:13.38909309Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-21T01:04:14.652103046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.9 kB" avg_comp="4.9 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-04-21T01:04:14.653976361Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19dad35a46e%3A19dad4d5459%3Acbe97912?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-04-21T01:04:14.653992538Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T01:04:14.65401178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T01:04:14.656904771Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T01:04:14.810968507Z caller=checkpoint.go:378 msg="writing series" size="977 kB"
level=debug ts=2026-04-21T01:04:14.810992509Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=7
level=info ts=2026-04-21T01:04:15.767161544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T01:04:15.770790531Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-21T01:04:16.280172161Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T01:04:17.310535483Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:04:17.648105947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T01:04:17.651614134Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-04-21T01:04:17.651656221Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T01:04:17.651677525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T01:04:17.654850767Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T01:04:18.560388446Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-21T01:04:18.560412895Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=1
level=info ts=2026-04-21T01:04:19.151493135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T01:04:19.155108023Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T01:04:19.357832113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T01:04:19.361587366Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T01:04:20.068255953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9b2c21dd0f9b9a09 immediate=false num_chunks=1 total_comp="642 kB" avg_comp="642 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" full=1 labels="{stream=\"build-e2e-lzeq\"}"
level=error ts=2026-04-21T01:04:20.07301367Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9b2c21dd0f9b9a09/19dad4476d4%3A19dad54bae0%3A7765f6b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lzeq\"}"
level=info ts=2026-04-21T01:04:22.30985833Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T01:04:22.309895744Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T01:04:22.309906983Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T01:04:22.30990268Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:04:22.310107546Z caller=checkpoint.go:378 msg="writing series" size="224 kB"
level=info ts=2026-04-21T01:04:22.536076405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T01:04:22.540109451Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T01:04:23.943271628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T01:04:23.947176524Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T01:04:25.554252531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T01:04:25.562325469Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T01:04:26.060907938Z caller=checkpoint.go:378 msg="writing series" size="161 kB"
level=info ts=2026-04-21T01:04:26.177194032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="23 MB" avg_comp="2.6 MB" total_uncomp="192 MB" avg_uncomp="21 MB" full=7 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T01:04:26.184310584Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-21T01:04:27.310233067Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T01:04:28.972030498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:53252"
level=debug ts=2026-04-21T01:04:29.810723438Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=info ts=2026-04-21T01:04:30.134368632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T01:04:30.138163915Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-04-21T01:04:32.309503204Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:04:33.560608056Z caller=checkpoint.go:378 msg="writing series" size="199 kB"
level=info ts=2026-04-21T01:04:34.32167177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T01:04:34.325191048Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T01:04:34.699505634Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T01:04:36.495386845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="8.2 kB" avg_comp="8.2 kB" total_uncomp="53 kB" avg_uncomp="53 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T01:04:36.500962463Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dad35c515%3A19dad58d1c0%3Acc49151d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-04-21T01:04:37.250534124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T01:04:37.310436682Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:04:37.337613754Z caller=checkpoint.go:378 msg="writing series" size="23 MB"
level=debug ts=2026-04-21T01:04:37.337659354Z caller=checkpoint.go:390 msg="flushing series" totalSize="24 MB" series=5
level=info ts=2026-04-21T01:04:40.863375521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T01:04:40.868746395Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T01:04:41.061110091Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=info ts=2026-04-21T01:04:41.250690767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="126 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T01:04:41.258378287Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T01:04:42.290326104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T01:04:42.295700401Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-21T01:04:42.309813609Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:04:44.810810066Z caller=checkpoint.go:378 msg="writing series" size="1.7 MB"
level=debug ts=2026-04-21T01:04:44.810837783Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.8 MB" series=2
level=info ts=2026-04-21T01:04:45.8136769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T01:04:45.817676182Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-21T01:04:47.310441936Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:04:48.56064934Z caller=checkpoint.go:378 msg="writing series" size="293 kB"
level=info ts=2026-04-21T01:04:51.849357542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T01:04:51.85335401Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T01:04:51.853374438Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T01:04:51.853395205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T01:04:51.85660894Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T01:04:52.286371907Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T01:04:52.286733731Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=info ts=2026-04-21T01:04:52.310202699Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T01:04:52.310207968Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:04:52.310238124Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T01:04:52.31024621Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T01:04:52.310274144Z caller=checkpoint.go:378 msg="writing series" size="9.2 kB"
level=info ts=2026-04-21T01:04:53.106507827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T01:04:53.111322642Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T01:04:53.48845612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T01:04:53.492417227Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-21T01:04:56.060010098Z caller=checkpoint.go:378 msg="writing series" size="83 kB"
level=info ts=2026-04-21T01:04:56.208675893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T01:04:56.212772678Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T01:04:56.750388311Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T01:04:57.310072094Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T01:04:58.973667716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:33916"
level=info ts=2026-04-21T01:04:59.65932892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T01:04:59.662890923Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T01:04:59.75152756Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T01:04:59.810821518Z caller=checkpoint.go:378 msg="writing series" size="277 kB"
level=info ts=2026-04-21T01:05:00.69932781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T01:05:00.702760743Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-04-21T01:05:00.702781705Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T01:05:00.702801779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T01:05:00.705540709Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-21T01:05:02.310220904Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:05:02.681463278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T01:05:02.684844823Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T01:05:03.560481816Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=info ts=2026-04-21T01:05:06.211250217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T01:05:06.214859707Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T01:05:06.214880474Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T01:05:06.214903011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T01:05:06.217895644Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T01:05:06.586213154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="23 MB" avg_comp="2.6 MB" total_uncomp="192 MB" avg_uncomp="21 MB" full=7 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T01:05:06.594704878Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T01:05:06.594734458Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T01:05:06.594775787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="23 MB" avg_comp="2.6 MB" total_uncomp="192 MB" avg_uncomp="21 MB" full=7 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T01:05:06.601589094Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T01:05:06.865378513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T01:05:06.868881215Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-04-21T01:05:07.252534151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-6gxwd-f2095043 10.128.0.86:7946"
level=warn ts=2026-04-21T01:05:07.309530137Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:05:07.310678269Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=info ts=2026-04-21T01:05:10.19855978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T01:05:10.202104732Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T01:05:11.060302083Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=info ts=2026-04-21T01:05:11.59923341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T01:05:11.603104605Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T01:05:11.603124062Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T01:05:11.603144504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T01:05:11.606182162Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T01:05:11.964187296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T01:05:11.967763944Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-21T01:05:12.310156393Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:05:14.013749089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T01:05:14.020881315Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T01:05:14.389224783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T01:05:14.393736259Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-21T01:05:14.810809305Z caller=checkpoint.go:378 msg="writing series" size="2.5 kB"
level=info ts=2026-04-21T01:05:15.440365902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9b2c21dd0f9b9a09 immediate=false num_chunks=1 total_comp="642 kB" avg_comp="642 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" full=1 labels="{stream=\"build-e2e-lzeq\"}"
level=error ts=2026-04-21T01:05:15.444766887Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9b2c21dd0f9b9a09/19dad4476d4%3A19dad54bae0%3A7765f6b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lzeq\"}"
level=error ts=2026-04-21T01:05:15.444788307Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T01:05:15.444810384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f8e39a532239c309 immediate=false num_chunks=1 total_comp="805 kB" avg_comp="805 kB" total_uncomp="5.5 MB" avg_uncomp="5.5 MB" idle=1 labels="{stream=\"build-e2e-xfsn\"}"
level=error ts=2026-04-21T01:05:15.459753844Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f8e39a532239c309/19dad443467%3A19dad58ae98%3Afdbef8fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xfsn\"}"
level=warn ts=2026-04-21T01:05:17.309824793Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:05:18.559867166Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=info ts=2026-04-21T01:05:18.907550652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="23 MB" avg_comp="2.6 MB" total_uncomp="192 MB" avg_uncomp="21 MB" full=7 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T01:05:18.914682893Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T01:05:21.26161587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T01:05:21.26689447Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T01:05:22.310399485Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T01:05:22.310402487Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:05:22.310429776Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T01:05:22.310438541Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T01:05:22.310431071Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=info ts=2026-04-21T01:05:23.439605427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T01:05:23.444099187Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T01:05:24.378612386Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T01:05:25.777767195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T01:05:25.781449272Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T01:05:26.060417506Z caller=checkpoint.go:378 msg="writing series" size="269 kB"
level=debug ts=2026-04-21T01:05:26.060447137Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-04-21T01:05:27.310461774Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T01:05:28.975599061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:54150"
level=debug ts=2026-04-21T01:05:29.810340323Z caller=checkpoint.go:378 msg="writing series" size="102 kB"
level=info ts=2026-04-21T01:05:30.66177185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T01:05:30.666759335Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T01:05:31.120640538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="126 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T01:05:31.127892969Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T01:05:32.310235162Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:05:33.56018991Z caller=checkpoint.go:378 msg="writing series" size="8.7 kB"
level=info ts=2026-04-21T01:05:34.373732944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f8e39a532239c309 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="5.5 MB" avg_uncomp="5.5 MB" full=1 labels="{stream=\"build-e2e-xfsn\"}"
level=error ts=2026-04-21T01:05:34.378490972Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f8e39a532239c309/19dad443467%3A19dad58ae98%3Afdbef8fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xfsn\"}"
level=info ts=2026-04-21T01:05:35.141582614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="8.2 kB" avg_comp="8.2 kB" total_uncomp="53 kB" avg_uncomp="53 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T01:05:35.145774937Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dad35c515%3A19dad58d1c0%3Acc49151d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-04-21T01:05:37.254361229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T01:05:37.310477392Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:05:37.310665434Z caller=checkpoint.go:378 msg="writing series" size="349 kB"
level=info ts=2026-04-21T01:05:37.696941975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T01:05:37.704669586Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T01:05:41.060810849Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-04-21T01:05:41.476217604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T01:05:41.480206543Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T01:05:41.689925221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T01:05:41.693539067Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T01:05:41.885557323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T01:05:41.888912539Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T01:05:41.974977588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T01:05:41.980300923Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-21T01:05:42.31011242Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:05:44.810814017Z caller=checkpoint.go:378 msg="writing series" size="192 kB"
level=info ts=2026-04-21T01:05:45.115606217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T01:05:45.121492256Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-04-21T01:05:45.121515556Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T01:05:45.121542438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T01:05:45.125126945Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-04-21T01:05:47.30974404Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:05:48.560776484Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=info ts=2026-04-21T01:05:52.286419294Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T01:05:52.28643535Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.85:11211 count=1
level=warn ts=2026-04-21T01:05:52.310009297Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:05:52.310009118Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-04-21T01:05:52.310007881Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T01:05:52.310034843Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T01:05:52.310045659Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T01:05:52.848812703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="490 kB" avg_comp="490 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T01:05:52.853222849Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dad35220e%3A19dad6215f4%3A340291df?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T01:05:53.883033197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T01:05:53.887517854Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dad35c515%3A19dad35c517%3Aea8ac59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T01:05:54.10617733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="23 MB" avg_comp="2.6 MB" total_uncomp="192 MB" avg_uncomp="21 MB" full=7 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T01:05:54.11355382Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-21T01:05:55.108468952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:57828"
level=info ts=2026-04-21T01:05:55.752910036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T01:05:55.756515178Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dad35a46e%3A19dad35c517%3Ae4c18105?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T01:05:56.052346253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T01:05:56.056318606Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-21T01:05:56.060668799Z caller=checkpoint.go:378 msg="writing series" size="555 kB"
level=debug ts=2026-04-21T01:05:56.060690067Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=8
level=warn ts=2026-04-21T01:05:57.310568539Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:05:58.44534377Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T01:05:59.013145074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T01:05:59.017200617Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dad35a46e%3A19dad35ce04%3A74ab4fdc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T01:05:59.102456597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T01:05:59.105844697Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dad35a494%3A19dad35a495%3Abd8723f6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T01:05:59.766694675Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T01:05:59.817928366Z caller=checkpoint.go:378 msg="writing series" size="15 MB"
level=debug ts=2026-04-21T01:05:59.817954321Z caller=checkpoint.go:390 msg="flushing series" totalSize="15 MB" series=1
level=info ts=2026-04-21T01:06:02.150087511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T01:06:02.153575078Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dad35a46e%3A19dad35c516%3A18f06248?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T01:06:02.298106141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T01:06:02.304166197Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dad3524b8%3A19dad63af05%3Ad8ac6a5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T01:06:02.310309282Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:06:03.560793501Z caller=checkpoint.go:378 msg="writing series" size="657 B"
level=info ts=2026-04-21T01:06:06.923197386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T01:06:06.927090088Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dad43c31f%3A19dad451bfc%3Aeb429069?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-21T01:06:07.084976492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T01:06:07.090091379Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dad352ac4%3A19dad7c0e64%3Ad5012257?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-04-21T01:06:07.255926603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-zcpwd-1aaf214c 10.128.0.90:7946"
level=warn ts=2026-04-21T01:06:07.310441745Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:06:07.310845715Z caller=checkpoint.go:378 msg="writing series" size="615 kB"
level=info ts=2026-04-21T01:06:10.337911593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f8e39a532239c309 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="5.5 MB" avg_uncomp="5.5 MB" full=1 labels="{stream=\"build-e2e-xfsn\"}"
level=error ts=2026-04-21T01:06:10.343344757Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f8e39a532239c309/19dad443467%3A19dad58ae98%3Afdbef8fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-xfsn\"}"
level=debug ts=2026-04-21T01:06:11.060327663Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-21T01:06:12.31028777Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:06:14.810199344Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
level=info ts=2026-04-21T01:06:17.047710635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T01:06:17.052442632Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dad351fd6%3A19dad587bd1%3A3ce7c207?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-21T01:06:17.309851577Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:06:18.560757787Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=info ts=2026-04-21T01:06:18.680211546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="1.9 MB" avg_uncomp="1.9 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T01:06:18.68446033Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dad351fd6%3A19dad74906b%3A9ea8b90?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-04-21T01:06:22.309579204Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:06:22.309612026Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T01:06:22.309647709Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T01:06:22.309655775Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T01:06:22.31077761Z caller=checkpoint.go:378 msg="writing series" size="194 kB"
level=info ts=2026-04-21T01:06:25.185221112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T01:06:25.190113795Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dad3520ea%3A19dad35c516%3A9485acf7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T01:06:26.06002903Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=warn ts=2026-04-21T01:06:27.309891478Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T01:06:28.229289081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T01:06:28.233120909Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dad35a46e%3A19dad35c516%3A9db621b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T01:06:28.299250159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="8.2 kB" avg_comp="8.2 kB" total_uncomp="53 kB" avg_uncomp="53 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T01:06:28.302778911Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dad35c515%3A19dad58d1c0%3Acc49151d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-21T01:06:28.450050538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T01:06:28.453602181Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dad351fe5%3A19dad35c516%3Ad3705fa9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T01:06:29.727180329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="126 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T01:06:29.734006561Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dad35238f%3A19dad35a42a%3Ab57ad6c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T01:06:29.810339318Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=debug ts=2026-04-21T01:06:30.543411302Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T01:06:32.309566011Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T01:06:33.560700204Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-04-21T01:06:34.381342749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T01:06:34.387133652Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dad352448%3A19dad6e8088%3Ac4fbf2b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T01:06:36.537401985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T01:06:36.541161105Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dad35d675%3A19dad360786%3A901f3793?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T01:06:37.129759711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="23 MB" avg_comp="2.6 MB" total_uncomp="192 MB" avg_uncomp="21 MB" full=7 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T01:06:37.137348995Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dad352d19%3A19dad444f85%3A92c82bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-21T01:06:37.257859594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-13f4eec1 10.130.0.67:7946"
level=debug ts=2026-04-21T01:06:37.310395597Z caller=checkpoint.go:378 msg="writing series" size="817 B"
level=warn ts=2026-04-21T01:06:37.310397789Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"