level=info ts=2026-05-12T21:27:39.404428055Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-12T21:27:39.404459774Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-05-12T21:27:39.404750393Z caller=modules.go:2203 msg="initializing ingester query tags interceptors"
level=info ts=2026-05-12T21:27:39.405539308Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-12T21:27:39.405906794Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-12T21:27:39.405974147Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.77 inf=eth0
level=info ts=2026-05-12T21:27:39.406494131Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-529f4c4f
ts=2026-05-12T21:27:39.406562058Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-12T21:27:39.406573979Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.77 advertisePort=7946
level=debug ts=2026-05-12T21:27:39.406592158Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.77 advertisePort=7946
level=info ts=2026-05-12T21:27:39.406703802Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-05-12T21:27:39.4095394Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-05-12T21:27:39.409559763Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
level=info ts=2026-05-12T21:27:39.410696302Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-05-12T21:27:39.410773008Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
ts=2026-05-12T21:27:39.411087744Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-12T21:27:39.411100072Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-12T21:27:39.411109173Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=4.407279ms
level=error ts=2026-05-12T21:27:39.411123344Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-12T21:27:39.411130147Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-05-12T21:27:39.411202644Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-12T21:27:39.411225403Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-12T21:27:39.411223068Z
level=debug ts=2026-05-12T21:27:39.411252053Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-05-12T21:27:39.411288113Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-05-12T21:27:39.411338173Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-12T21:27:39.41134696Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-12T21:27:39.411345337Z
level=debug ts=2026-05-12T21:27:39.411354178Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=debug ts=2026-05-12T21:27:39.412673036Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-12T21:27:39.412741486Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.128.0.77 inf=eth0
ts=2026-05-12T21:27:39.41334624Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T21:27:39.413357596Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-12T21:27:39.426101267Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-05-12T21:27:39.426161291Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=info ts=2026-05-12T21:27:39.426174291Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-12T21:27:39.426168875Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=debug ts=2026-05-12T21:27:39.426177168Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-12T21:27:39.426190465Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-12T21:27:39.426188843Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-05-12T21:27:39.426277106Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=info ts=2026-05-12T21:27:39.426277008Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-12T21:27:39.426293318Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-05-12T21:27:39.426318594Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-05-12T21:27:39.426401029Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-05-12T21:27:39.426404934Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-05-12T21:27:39.426552267Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-05-12T21:27:39.426563116Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-05-12T21:27:39.426566223Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-05-12T21:27:39.426599757Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-05-12T21:27:39.426628894Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-05-12T21:27:39.426632532Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=info ts=2026-05-12T21:27:39.4266352Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-12T21:27:39.426667572Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=debug ts=2026-05-12T21:27:39.426745024Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-05-12T21:27:39.426757116Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-05-12T21:27:39.426760472Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=info ts=2026-05-12T21:27:39.426763276Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-05-12T21:27:39.426774796Z caller=ingester.go:571 component=ingester msg="recovering from checkpoint"
level=info ts=2026-05-12T21:27:39.426784026Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T21:27:39.426823275Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=18.445µs success=true
level=info ts=2026-05-12T21:27:39.426835394Z caller=recovery.go:45 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-05-12T21:27:39.426896259Z caller=ingester.go:587 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=122.987µs errors=false
level=info ts=2026-05-12T21:27:39.426906519Z caller=ingester.go:593 component=ingester msg="recovering from WAL"
level=info ts=2026-05-12T21:27:39.427012918Z caller=ingester.go:609 component=ingester msg="WAL segment recovery finished" elapsed=239.19µs errors=false
level=info ts=2026-05-12T21:27:39.42702446Z caller=ingester.go:557 component=ingester msg="closing recoverer"
level=info ts=2026-05-12T21:27:39.427029704Z caller=ingester.go:565 component=ingester msg="WAL recovery finished" time=256.903µs
level=info ts=2026-05-12T21:27:39.427047252Z caller=wal.go:173 msg=started component=wal
level=info ts=2026-05-12T21:27:39.42707907Z caller=lifecycler.go:694 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-12T21:27:39.427084821Z caller=ingester.go:778 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=167.656377ms
level=info ts=2026-05-12T21:27:39.427097588Z caller=lifecycler.go:721 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-05-12T21:27:39.427165046Z caller=ingester.go:473 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-05-12T21:27:39.42717518Z caller=loki.go:615 msg="Loki started" startup_time=84.817654ms
level=debug ts=2026-05-12T21:27:39.4274407Z caller=lifecycler.go:558 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-05-12T21:27:39.427448365Z caller=lifecycler.go:562 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=debug ts=2026-05-12T21:27:39.427575693Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-05-12T21:27:41.344412439Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T21:27:41.344435464Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-12T21:27:44.427433547Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:27:44.965368356Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T21:27:44.9653983Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-12T21:27:49.428220776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:27:49.855163413Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T21:27:49.855186987Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-12T21:27:54.019474587Z caller=logging.go:137 msg="GET /ready (503) 93.503µs"
level=warn ts=2026-05-12T21:27:54.427601019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:27:59.427685332Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:28:01.070747018Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.76:7946: dial tcp 10.128.0.76:7946: connect: connection refused"
level=warn ts=2026-05-12T21:28:01.070769743Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="1 error occurred:\n\t* failed to join 10.128.0.76:7946: dial tcp 10.128.0.76:7946: connect: connection refused\n\n"
level=debug ts=2026-05-12T21:28:04.017787006Z caller=logging.go:137 msg="GET /ready (503) 70.517µs"
level=warn ts=2026-05-12T21:28:04.428044331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:28:09.427231825Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:28:09.42723062Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-12T21:28:09.427265201Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-05-12T21:28:09.427269934Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:28:11.810223841Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:28:14.428056906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:28:19.427251946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:28:24.428272489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:28:25.692503606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:47138"
ts=2026-05-12T21:28:26.161876246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:49522"
level=warn ts=2026-05-12T21:28:29.427715405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:28:29.435899272Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:42854"
ts=2026-05-12T21:28:31.356879493Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.76:7946: dial tcp 10.128.0.76:7946: connect: connection refused"
ts=2026-05-12T21:28:31.356996051Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.77:7946"
ts=2026-05-12T21:28:31.357049639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:33480"
ts=2026-05-12T21:28:31.357525457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.74:7946"
ts=2026-05-12T21:28:31.359010496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.59:7946"
ts=2026-05-12T21:28:31.361598212Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.58:7946: dial tcp 10.130.0.58:7946: connect: connection refused"
ts=2026-05-12T21:28:31.362648758Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.60:7946: dial tcp 10.130.0.60:7946: connect: connection refused"
ts=2026-05-12T21:28:31.363651914Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.61:7946"
level=info ts=2026-05-12T21:28:31.364854022Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=4 elapsed_time=51.953719364s
level=warn ts=2026-05-12T21:28:34.428118753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:28:36.445722833Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T21:28:39.410828086Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:28:39.411005621Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T21:28:39.427290678Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:28:39.427307771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:28:39.42732369Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:28:39.427331087Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:28:41.266028843Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-12T21:28:44.427268922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:28:49.427795301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:28:49.798017497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T21:28:50.856386325Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:40020"
ts=2026-05-12T21:28:51.83450274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:44580"
level=warn ts=2026-05-12T21:28:54.428017847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:28:57.778343858Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:32908"
level=warn ts=2026-05-12T21:28:59.427418073Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:29:03.278738161Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-12T21:29:04.42754271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:29:09.427382186Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:29:09.427410538Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:29:09.427419103Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:29:09.427432302Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:29:10.199747314Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T21:29:10.199798922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:29:10.209695695Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:29:14.427347691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:29:19.428053167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:29:19.801397758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=info ts=2026-05-12T21:29:21.899897776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:29:21.907856893Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:29:24.427885531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:29:24.999792264Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-12T21:29:29.427785117Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:29:34.427364209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:29:39.41074876Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T21:29:39.411806856Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-12T21:29:39.428090812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:29:39.428095067Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:29:39.428113105Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:29:39.428122562Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:29:42.80306806Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:29:44.42762331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:29:49.427558753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:29:49.80363685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T21:29:50.08806889Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-12T21:29:54.428203901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:29:57.784726013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:36802"
level=info ts=2026-05-12T21:29:58.421199803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:29:58.428017324Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:29:59.427683366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:30:04.427626264Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:30:09.427679223Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:30:09.427705666Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:30:09.427713252Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:30:09.427728211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:30:14.42789374Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:30:14.434072248Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:30:17.618899632Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-12T21:30:17.620515128Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T21:30:19.428271493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:30:19.80440598Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T21:30:21.842173432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:34416"
level=warn ts=2026-05-12T21:30:24.428266124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:30:27.787407949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:37530"
level=warn ts=2026-05-12T21:30:29.427210475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:30:34.427354065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:30:39.411419674Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:30:39.41219576Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-12T21:30:39.412255137Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976245 n=1
level=debug ts=2026-05-12T21:30:39.41232404Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976245 n=1
level=debug ts=2026-05-12T21:30:39.412462837Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=warn ts=2026-05-12T21:30:39.427753904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:30:39.427757966Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:30:39.427775118Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:30:39.427784657Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:30:44.42827321Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:30:46.018889929Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T21:30:46.075996409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:30:46.082953481Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:30:49.427508238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:30:49.806230868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T21:30:51.84335187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:34790"
level=warn ts=2026-05-12T21:30:54.427763723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:30:59.427612023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:30:59.841138171Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-12T21:31:04.42814946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:31:09.427899576Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:31:09.427929831Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:31:09.427940134Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:31:09.427900685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:31:13.006316988Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:31:14.427918103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:31:19.427536875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:31:19.808683387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=warn ts=2026-05-12T21:31:24.427727399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:31:27.092363283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:31:27.102550265Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-12T21:31:27.793749085Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:40526"
level=warn ts=2026-05-12T21:31:29.427940625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:31:34.427637599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:31:39.411318388Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:31:39.411854259Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T21:31:39.427173701Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:31:39.427191841Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:31:39.427201133Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:31:39.427205157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:31:39.510588656Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:31:44.427676356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:31:49.427490817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:31:49.811323914Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=warn ts=2026-05-12T21:31:54.428237921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:31:57.796530682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:57010"
level=warn ts=2026-05-12T21:31:59.427543628Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:32:01.603186792Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-12T21:32:04.427563016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:32:09.42768254Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:32:09.427707421Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:32:09.427715274Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:32:09.427715491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:32:12.643936998Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:32:14.428256143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:32:18.150922852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:32:18.159517981Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:32:19.427751239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:32:19.814980207Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T21:32:21.850875627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:46028"
level=warn ts=2026-05-12T21:32:24.427235983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:32:29.428160984Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:32:34.427878388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:32:36.878729625Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T21:32:39.411603649Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:32:39.412222786Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=warn ts=2026-05-12T21:32:39.427422254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:32:39.427440737Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-12T21:32:39.427448654Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-12T21:32:39.427462943Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:32:39.427472713Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T21:32:39.427578917Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-12T21:32:39.42758793Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T21:32:39.427603334Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=10.776µs success=true
level=debug ts=2026-05-12T21:32:39.427857491Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=debug ts=2026-05-12T21:32:43.783650439Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-05-12T21:32:44.427858499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:32:48.138368037Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-12T21:32:49.427767328Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:32:49.817379266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=debug ts=2026-05-12T21:32:52.492547621Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=warn ts=2026-05-12T21:32:54.427226104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:32:56.848127924Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
ts=2026-05-12T21:32:57.802311584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:47328"
level=warn ts=2026-05-12T21:32:59.427263362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:33:01.202283119Z caller=checkpoint.go:372 msg="writing series" size="175 kB"
level=warn ts=2026-05-12T21:33:04.42780408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:33:05.233284216Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:33:05.557506778Z caller=checkpoint.go:372 msg="writing series" size="159 kB"
level=info ts=2026-05-12T21:33:07.972340901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:33:07.97881316Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T21:33:09.427977258Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:33:09.427979365Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:33:09.428003779Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:33:09.42801115Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:33:09.912384946Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-12T21:33:11.289733058Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-12T21:33:14.268716365Z caller=checkpoint.go:372 msg="writing series" size="458 kB"
level=warn ts=2026-05-12T21:33:14.427531657Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:33:18.622283529Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-12T21:33:19.42734058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:33:19.819627322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T21:33:22.976929643Z caller=checkpoint.go:372 msg="writing series" size="165 kB"
level=debug ts=2026-05-12T21:33:22.976949979Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=11
level=warn ts=2026-05-12T21:33:24.427931464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:33:27.331184694Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-12T21:33:29.427335793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:33:31.686682373Z caller=checkpoint.go:372 msg="writing series" size="480 kB"
level=warn ts=2026-05-12T21:33:34.427851783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:33:36.041429653Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=info ts=2026-05-12T21:33:39.411558982Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:33:39.412038038Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T21:33:39.427246238Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:33:39.427255671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:33:39.427265273Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:33:39.427272526Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:33:40.128334023Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:33:40.39645823Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=warn ts=2026-05-12T21:33:44.427701257Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:33:44.750717272Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=debug ts=2026-05-12T21:33:49.109443328Z caller=checkpoint.go:372 msg="writing series" size="5.8 MB"
level=debug ts=2026-05-12T21:33:49.109597134Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.4 MB" series=6
level=warn ts=2026-05-12T21:33:49.428096083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:33:49.664899726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:33:49.672860686Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-12T21:33:49.820374023Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T21:33:53.460316625Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=warn ts=2026-05-12T21:33:54.428157718Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:33:57.808414997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:40094"
level=debug ts=2026-05-12T21:33:57.815799808Z caller=checkpoint.go:372 msg="writing series" size="77 kB"
level=warn ts=2026-05-12T21:33:59.427345973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:34:02.170657143Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-12T21:34:02.170677644Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=warn ts=2026-05-12T21:34:04.428229006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:34:04.679471574Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-12T21:34:06.525284865Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-12T21:34:07.546311842Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T21:34:09.427550857Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:34:09.427574614Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-12T21:34:09.427571126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:34:09.427582175Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:34:10.880220991Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-12T21:34:14.427548283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:34:15.235136342Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-12T21:34:19.428250141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:34:19.589563327Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
ts=2026-05-12T21:34:19.822090839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T21:34:20.882568246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:58478"
level=debug ts=2026-05-12T21:34:23.944355473Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-12T21:34:24.427761442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:34:27.81088316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:48966"
level=debug ts=2026-05-12T21:34:28.299332296Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-12T21:34:29.427536273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:34:32.654146966Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-12T21:34:33.123510868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:34:33.134167639Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:34:33.55645719Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:34:34.427866443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:34:37.008801821Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-05-12T21:34:39.410492998Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T21:34:39.410801349Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T21:34:39.428145383Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:34:39.428160786Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:34:39.428168248Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:34:39.428217863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:34:41.364368029Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=warn ts=2026-05-12T21:34:44.428033386Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:34:45.718342426Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-12T21:34:49.427979451Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:34:49.823980003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T21:34:50.073088493Z caller=checkpoint.go:372 msg="writing series" size="218 kB"
ts=2026-05-12T21:34:51.864663417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:40668"
level=warn ts=2026-05-12T21:34:54.428212835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:34:54.428218729Z caller=checkpoint.go:372 msg="writing series" size="514 B"
ts=2026-05-12T21:34:57.813667237Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:37950"
level=debug ts=2026-05-12T21:34:58.783291705Z caller=checkpoint.go:372 msg="writing series" size="1.6 kB"
level=warn ts=2026-05-12T21:34:59.427233279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:35:03.138358158Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=warn ts=2026-05-12T21:35:04.428198193Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:35:05.578010686Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:35:05.735565555Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-12T21:35:05.737150516Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-12T21:35:07.493209498Z caller=checkpoint.go:372 msg="writing series" size="333 kB"
level=info ts=2026-05-12T21:35:09.428158722Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:35:09.428182755Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:35:09.428189965Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:35:09.428205195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:35:11.847670668Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=debug ts=2026-05-12T21:35:11.847692841Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=16
level=warn ts=2026-05-12T21:35:14.427674291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:35:16.202343888Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-05-12T21:35:19.42720406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:35:19.82467301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T21:35:20.55770688Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-12T21:35:24.428127295Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:35:24.91237502Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-12T21:35:27.912730126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:35:27.920112845Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:35:27.920134917Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T21:35:27.920156494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:35:27.926263861Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:35:29.266952263Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=warn ts=2026-05-12T21:35:29.428243072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:35:33.621567754Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-12T21:35:34.428139768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:35:37.97654368Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-12T21:35:39.410838687Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:35:39.412856008Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T21:35:39.428146035Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:35:39.428168304Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:35:39.428178585Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:35:39.428196623Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:35:40.138831156Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:35:42.332095834Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=info ts=2026-05-12T21:35:43.235634539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:35:43.244491917Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:35:44.427934798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:35:46.686621453Z caller=checkpoint.go:372 msg="writing series" size="9.0 kB"
level=warn ts=2026-05-12T21:35:49.428230921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:35:49.827228883Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T21:35:50.892241233Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:55396"
level=debug ts=2026-05-12T21:35:51.041078399Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=warn ts=2026-05-12T21:35:54.42734445Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:35:55.396164137Z caller=checkpoint.go:372 msg="writing series" size="358 kB"
ts=2026-05-12T21:35:57.818055473Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:56142"
level=warn ts=2026-05-12T21:35:59.427938556Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:35:59.750800852Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=debug ts=2026-05-12T21:36:04.105632339Z caller=checkpoint.go:372 msg="writing series" size="247 kB"
level=debug ts=2026-05-12T21:36:04.105654086Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
level=warn ts=2026-05-12T21:36:04.428110466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:36:05.105721985Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T21:36:07.584904164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:36:07.591577308Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:36:08.460200683Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-12T21:36:09.427840139Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:36:09.427865875Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:36:09.427873546Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:36:09.42788161Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:36:12.815811354Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-12T21:36:14.427881724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:36:15.524074843Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:36:17.170323017Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-12T21:36:19.427432388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:36:19.828688391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T21:36:21.525422699Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-12T21:36:24.428175659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:36:25.879679583Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
ts=2026-05-12T21:36:27.820774469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:58184"
level=warn ts=2026-05-12T21:36:29.427426944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:36:30.235046244Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=warn ts=2026-05-12T21:36:34.427613163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:36:34.59017Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=debug ts=2026-05-12T21:36:38.944634282Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-12T21:36:39.41178284Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:36:39.412313005Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T21:36:39.427651527Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:36:39.427668718Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:36:39.42767643Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:36:39.427677689Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:36:43.299691829Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-12T21:36:44.427632258Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:36:47.653906537Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=debug ts=2026-05-12T21:36:48.000725883Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T21:36:48.281762965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:36:48.289747779Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:36:48.395062908Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:36:49.428001476Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:36:49.83068424Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=debug ts=2026-05-12T21:36:52.009417038Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=warn ts=2026-05-12T21:36:54.427723716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:36:56.363607933Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
ts=2026-05-12T21:36:57.824431543Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:50986"
level=warn ts=2026-05-12T21:36:59.427957701Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:37:00.718893855Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-12T21:37:04.42801033Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:37:05.07379178Z caller=checkpoint.go:372 msg="writing series" size="282 kB"
level=debug ts=2026-05-12T21:37:05.073817753Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=14
level=warn ts=2026-05-12T21:37:09.427811569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:37:09.427822323Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:37:09.427841913Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:37:09.427849965Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T21:37:09.433077327Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-12T21:37:09.436566715Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008802932s
level=warn ts=2026-05-12T21:37:14.428130473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:37:19.427264206Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:37:19.718604974Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-12T21:37:19.834135525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=warn ts=2026-05-12T21:37:24.427958909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:37:27.827119216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:54692"
level=warn ts=2026-05-12T21:37:29.427705029Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:37:30.012191766Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T21:37:33.663596308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:37:33.672233651Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:37:34.4274983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:37:39.410805244Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:37:39.410986451Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T21:37:39.427375602Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:37:39.427393648Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:37:39.427401289Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T21:37:39.427410378Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-12T21:37:39.427438026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:37:39.427527562Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-12T21:37:39.427534466Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T21:37:39.427552558Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.45µs success=true
level=debug ts=2026-05-12T21:37:39.42776159Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=debug ts=2026-05-12T21:37:43.78274866Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-05-12T21:37:44.427214584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:37:48.137464426Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-12T21:37:49.165064786Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:37:49.427739747Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:37:49.837249396Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T21:37:50.902687948Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:34622"
level=debug ts=2026-05-12T21:37:52.492584391Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=warn ts=2026-05-12T21:37:54.427596802Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:37:56.847712734Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-12T21:37:59.427337387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:38:01.202548563Z caller=checkpoint.go:372 msg="writing series" size="306 kB"
level=warn ts=2026-05-12T21:38:04.427348203Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:38:05.557466768Z caller=checkpoint.go:372 msg="writing series" size="299 kB"
level=debug ts=2026-05-12T21:38:08.515659881Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T21:38:09.427288892Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:38:09.427313036Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:38:09.427326389Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:38:09.427334262Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:38:09.911967034Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-12T21:38:14.266985757Z caller=checkpoint.go:372 msg="writing series" size="453 kB"
level=debug ts=2026-05-12T21:38:14.267003698Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=9
level=warn ts=2026-05-12T21:38:14.427952583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:38:18.621635033Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=debug ts=2026-05-12T21:38:19.17703387Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:38:19.427786618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:38:19.840444269Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T21:38:21.883523293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:36430"
level=debug ts=2026-05-12T21:38:22.97700312Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
level=info ts=2026-05-12T21:38:24.078767293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:38:24.086516269Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:38:24.427264438Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:38:27.331578112Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-12T21:38:29.427534083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:38:31.686362226Z caller=checkpoint.go:372 msg="writing series" size="452 kB"
level=warn ts=2026-05-12T21:38:34.428038248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:38:36.0413651Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=info ts=2026-05-12T21:38:39.41145248Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:38:39.411914204Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T21:38:39.427229785Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:38:39.427251496Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:38:39.427262251Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:38:39.427283206Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:38:40.396340265Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=debug ts=2026-05-12T21:38:43.41161143Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T21:38:43.413343174Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T21:38:44.427583427Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:38:44.750427717Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=debug ts=2026-05-12T21:38:45.407770701Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:38:49.109147111Z caller=checkpoint.go:372 msg="writing series" size="6.6 MB"
level=debug ts=2026-05-12T21:38:49.109170423Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.3 MB" series=8
level=warn ts=2026-05-12T21:38:49.427505175Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:38:49.844734786Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=debug ts=2026-05-12T21:38:53.460365969Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=warn ts=2026-05-12T21:38:54.428037241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:38:57.815909938Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=warn ts=2026-05-12T21:38:59.428049527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:39:02.170757945Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-12T21:39:02.170780552Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=warn ts=2026-05-12T21:39:04.427541936Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:39:06.525239133Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-12T21:39:09.427860678Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:39:09.42788946Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:39:09.427897423Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:39:09.427892732Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:39:10.879814587Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=debug ts=2026-05-12T21:39:12.487772624Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:39:14.428096674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:39:15.235181015Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-12T21:39:19.428142995Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:39:19.589580875Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
ts=2026-05-12T21:39:19.847517595Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=info ts=2026-05-12T21:39:22.115075419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:39:22.12606097Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:39:23.944037291Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-12T21:39:24.427999822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:39:27.572133976Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T21:39:28.299277097Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-12T21:39:29.427205173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:39:32.654080649Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-12T21:39:34.428140743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:39:37.008790692Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-05-12T21:39:37.571402363Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T21:39:39.411425884Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:39:39.411829418Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T21:39:39.427170727Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:39:39.427186806Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:39:39.427194522Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:39:39.427216003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:39:41.363479314Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=warn ts=2026-05-12T21:39:44.427725937Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:39:45.71836099Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-12T21:39:49.428137751Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:39:49.850728745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=debug ts=2026-05-12T21:39:50.073822664Z caller=checkpoint.go:372 msg="writing series" size="220 kB"
ts=2026-05-12T21:39:50.909020469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:56242"
level=warn ts=2026-05-12T21:39:54.428226861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:39:54.428254874Z caller=checkpoint.go:372 msg="writing series" size="514 B"
level=debug ts=2026-05-12T21:39:58.782932759Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=warn ts=2026-05-12T21:39:59.428085482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:40:03.13766844Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=warn ts=2026-05-12T21:40:04.427558657Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:40:04.600501503Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T21:40:04.667024202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:40:04.675272514Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:40:07.492912544Z caller=checkpoint.go:372 msg="writing series" size="329 kB"
level=debug ts=2026-05-12T21:40:07.492934362Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=15
level=info ts=2026-05-12T21:40:09.427912875Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:40:09.427944027Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:40:09.427954515Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:40:09.427973754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:40:10.726737883Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:40:11.847339643Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=warn ts=2026-05-12T21:40:14.427898976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:40:16.202017568Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-05-12T21:40:19.42797037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:40:19.853185967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T21:40:20.557694279Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-12T21:40:24.427352712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:40:24.912609574Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=debug ts=2026-05-12T21:40:29.266876338Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=warn ts=2026-05-12T21:40:29.428180206Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:40:33.621502307Z caller=checkpoint.go:372 msg="writing series" size="4.5 kB"
level=warn ts=2026-05-12T21:40:34.428156172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:40:37.977151874Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=info ts=2026-05-12T21:40:39.41181397Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:40:39.412683069Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T21:40:39.427964358Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:40:39.427982078Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:40:39.427989258Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:40:39.428016894Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:40:42.331410392Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=warn ts=2026-05-12T21:40:44.427228917Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:40:44.950693301Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:40:46.686134034Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-12T21:40:49.427496804Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:40:49.854803213Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T21:40:50.914827105Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:42794"
level=debug ts=2026-05-12T21:40:51.041589346Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=warn ts=2026-05-12T21:40:54.428131149Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:40:54.975971099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T21:40:55.396196027Z caller=checkpoint.go:372 msg="writing series" size="443 kB"
level=debug ts=2026-05-12T21:40:55.396215606Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-05-12T21:40:57.763077196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:40:57.769846074Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:40:59.427377223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:40:59.750586325Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
level=debug ts=2026-05-12T21:41:04.10588857Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
level=warn ts=2026-05-12T21:41:04.427343212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:41:08.460320597Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
level=info ts=2026-05-12T21:41:09.427742353Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:41:09.42777054Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:41:09.427777944Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:41:09.427789433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:41:12.815024439Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-12T21:41:14.42804465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:41:16.547943195Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:41:17.1706003Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-12T21:41:19.427504018Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:41:19.85560919Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T21:41:21.524761364Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-12T21:41:24.4276428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:41:25.879891482Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-12T21:41:29.427351224Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:41:30.234177598Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=debug ts=2026-05-12T21:41:33.630926625Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T21:41:34.427601104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:41:34.589895267Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=debug ts=2026-05-12T21:41:38.944739279Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-12T21:41:39.410806704Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:41:39.410808318Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=warn ts=2026-05-12T21:41:39.42722566Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:41:39.42723056Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:41:39.427243634Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:41:39.427251153Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:41:43.298795423Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-12T21:41:44.159430909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:41:44.16644254Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:41:44.166467906Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T21:41:44.166489867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:41:44.172967437Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:41:44.427415099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:41:45.40194195Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:41:47.653734897Z caller=checkpoint.go:372 msg="writing series" size="175 kB"
level=debug ts=2026-05-12T21:41:47.653756088Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
level=warn ts=2026-05-12T21:41:49.42726028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:41:49.857860577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T21:41:52.009374806Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=warn ts=2026-05-12T21:41:54.427841485Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:41:56.363368719Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-12T21:41:59.427434152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:42:00.71879119Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=info ts=2026-05-12T21:42:01.810079178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:42:01.820658629Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:42:04.427343842Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:42:05.073204883Z caller=checkpoint.go:372 msg="writing series" size="282 kB"
level=debug ts=2026-05-12T21:42:09.427733815Z caller=checkpoint.go:384 msg="flushing series" totalSize="497 kB" series=4
level=warn ts=2026-05-12T21:42:09.427755466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:42:09.427767319Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:42:09.427784729Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:42:09.4277923Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T21:42:09.433647211Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-12T21:42:09.435346196Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007676379s
level=debug ts=2026-05-12T21:42:12.921000454Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T21:42:12.922593343Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T21:42:14.427352567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:42:16.60099258Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:42:19.427931042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:42:19.860349751Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T21:42:20.920941313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:42276"
level=warn ts=2026-05-12T21:42:24.427846778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:42:24.50615311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:42:24.513766599Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:42:29.431116823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:42:34.428031475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:42:39.411723006Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:42:39.412517378Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T21:42:39.427913421Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-12T21:42:39.427927455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:42:39.427943855Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:42:39.427962316Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:42:39.427980073Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T21:42:39.428020444Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-12T21:42:39.428029215Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T21:42:39.42804685Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.662µs success=true
level=debug ts=2026-05-12T21:42:39.428238208Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=debug ts=2026-05-12T21:42:43.783441737Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-05-12T21:42:44.428184966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:42:45.584460377Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:42:48.138759934Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-12T21:42:49.427984291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:42:49.863379381Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T21:42:50.923812168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:46388"
ts=2026-05-12T21:42:51.90821413Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:48744"
level=debug ts=2026-05-12T21:42:52.493242679Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=debug ts=2026-05-12T21:42:53.278941859Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T21:42:54.427961363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:42:56.848550927Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-12T21:42:59.427527334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:43:01.202843148Z caller=checkpoint.go:372 msg="writing series" size="206 kB"
level=warn ts=2026-05-12T21:43:04.427518924Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:43:05.557623811Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=info ts=2026-05-12T21:43:09.42789438Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:43:09.427925532Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:43:09.427938614Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:43:09.427950345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:43:09.912783654Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-12T21:43:10.566992061Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:43:14.267600176Z caller=checkpoint.go:372 msg="writing series" size="569 kB"
level=debug ts=2026-05-12T21:43:14.267627047Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=warn ts=2026-05-12T21:43:14.427920763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:43:18.622277862Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=info ts=2026-05-12T21:43:19.309083238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:43:19.316533397Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:43:19.427771932Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:43:19.866603727Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=debug ts=2026-05-12T21:43:22.977836961Z caller=checkpoint.go:372 msg="writing series" size="323 kB"
level=warn ts=2026-05-12T21:43:24.427539692Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:43:27.331829084Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-12T21:43:29.427231484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:43:31.687263855Z caller=checkpoint.go:372 msg="writing series" size="284 kB"
level=warn ts=2026-05-12T21:43:34.427555956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:43:36.041347153Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=info ts=2026-05-12T21:43:39.411617763Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:43:39.412235017Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T21:43:39.427535874Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:43:39.427561163Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:43:39.427568569Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:43:39.427564924Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:43:40.396232078Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=debug ts=2026-05-12T21:43:41.754457236Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:43:44.427408855Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:43:44.751288252Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=debug ts=2026-05-12T21:43:49.11103698Z caller=checkpoint.go:372 msg="writing series" size="7.9 MB"
level=debug ts=2026-05-12T21:43:49.111057171Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.6 MB" series=8
level=warn ts=2026-05-12T21:43:49.428155005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:43:49.869284091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T21:43:50.487848767Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:43:50.930240929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:38154"
level=debug ts=2026-05-12T21:43:53.460741325Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=warn ts=2026-05-12T21:43:54.427662689Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:43:57.815723343Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=warn ts=2026-05-12T21:43:59.427802377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:44:02.171092499Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-12T21:44:02.17111311Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=warn ts=2026-05-12T21:44:04.428058384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:44:06.525502697Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-12T21:44:09.427176222Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:44:09.427200281Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:44:09.427207648Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:44:09.427206727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:44:10.40996351Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:44:10.880409101Z caller=checkpoint.go:372 msg="writing series" size="212 kB"
level=info ts=2026-05-12T21:44:14.155271847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:44:14.16327446Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:44:14.427892715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:44:15.235530752Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-12T21:44:19.427964906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:44:19.590365096Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
ts=2026-05-12T21:44:19.87162931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T21:44:20.933632139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:45388"
level=debug ts=2026-05-12T21:44:23.944403291Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-12T21:44:24.427881717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:44:27.857673392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:45344"
level=debug ts=2026-05-12T21:44:28.300065208Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-12T21:44:29.428176581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:44:32.655060309Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-12T21:44:34.427956479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:44:34.84275834Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:44:37.009198077Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-12T21:44:39.411117026Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:44:39.411687003Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T21:44:39.428119346Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:44:39.42814106Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:44:39.428148526Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:44:39.42818475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:44:41.36446558Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=debug ts=2026-05-12T21:44:43.458076394Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T21:44:44.427223618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:44:45.718733245Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-12T21:44:49.428260209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:44:49.87387873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T21:44:50.074056259Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=debug ts=2026-05-12T21:44:54.428258293Z caller=checkpoint.go:372 msg="writing series" size="514 B"
level=warn ts=2026-05-12T21:44:54.428220171Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:44:56.77532026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:44:56.785520231Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-12T21:44:57.861370219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:53840"
level=debug ts=2026-05-12T21:44:58.783460747Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-12T21:44:59.42799033Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:45:03.138365861Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=warn ts=2026-05-12T21:45:04.42825017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:45:05.572249884Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:45:07.493880743Z caller=checkpoint.go:372 msg="writing series" size="364 kB"
level=debug ts=2026-05-12T21:45:07.493967774Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=15
level=info ts=2026-05-12T21:45:09.427939791Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:45:09.42796544Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:45:09.427974506Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:45:09.427983829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:45:11.848216908Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=warn ts=2026-05-12T21:45:14.428010846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:45:16.203142204Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-05-12T21:45:19.427538885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:45:19.877195791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T21:45:19.941474601Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T21:45:20.557529755Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
ts=2026-05-12T21:45:20.938045789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:39310"
level=warn ts=2026-05-12T21:45:24.428019182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:45:24.913117174Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=debug ts=2026-05-12T21:45:29.267250338Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=warn ts=2026-05-12T21:45:29.427487859Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:45:33.622555888Z caller=checkpoint.go:372 msg="writing series" size="4.5 kB"
level=warn ts=2026-05-12T21:45:34.427607479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:45:34.789751343Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:45:37.976836175Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=info ts=2026-05-12T21:45:39.410973815Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:45:39.411647765Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=debug ts=2026-05-12T21:45:39.412919231Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-12T21:45:39.412961386Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976246 n=1
level=debug ts=2026-05-12T21:45:39.413003285Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976246 n=1
level=info ts=2026-05-12T21:45:39.427944205Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:45:39.427950646Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:45:39.427960976Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:45:39.427968752Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:45:42.332386803Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=info ts=2026-05-12T21:45:43.953075918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:45:43.961953729Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:45:44.42818758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:45:46.686545436Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=warn ts=2026-05-12T21:45:49.427777168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:45:49.880459177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=debug ts=2026-05-12T21:45:51.041642454Z caller=checkpoint.go:372 msg="writing series" size="83 kB"
ts=2026-05-12T21:45:51.923827168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:48992"
level=warn ts=2026-05-12T21:45:54.428024185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:45:55.396202763Z caller=checkpoint.go:372 msg="writing series" size="286 kB"
level=debug ts=2026-05-12T21:45:59.156432556Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T21:45:59.158230362Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T21:45:59.42798703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:45:59.751071604Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=debug ts=2026-05-12T21:46:04.106040894Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=debug ts=2026-05-12T21:46:04.106061435Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=13
level=warn ts=2026-05-12T21:46:04.427474394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:46:08.337401373Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:46:08.46105168Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=info ts=2026-05-12T21:46:09.427218429Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:46:09.427222272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:46:09.427243196Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:46:09.427251552Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:46:12.815429335Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=warn ts=2026-05-12T21:46:14.428283506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:46:17.170560543Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=warn ts=2026-05-12T21:46:19.427211877Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:46:19.883965994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T21:46:21.52553065Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
ts=2026-05-12T21:46:21.925274141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:52094"
level=warn ts=2026-05-12T21:46:24.428070724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:46:25.880631894Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
ts=2026-05-12T21:46:27.869108046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:40726"
level=warn ts=2026-05-12T21:46:29.42725068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:46:30.234943346Z caller=checkpoint.go:372 msg="writing series" size="4.5 kB"
level=info ts=2026-05-12T21:46:32.874351677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:46:32.881323879Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:46:33.866660564Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T21:46:34.427723768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:46:34.590153854Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=debug ts=2026-05-12T21:46:38.944725216Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-12T21:46:39.410845399Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:46:39.410862398Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T21:46:39.42745817Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:46:39.427485035Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:46:39.427494599Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:46:39.427531701Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:46:41.685152096Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:46:43.300313749Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=warn ts=2026-05-12T21:46:44.427537965Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:46:47.654998352Z caller=checkpoint.go:372 msg="writing series" size="175 kB"
level=warn ts=2026-05-12T21:46:49.42758626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:46:49.886567019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T21:46:50.948585266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:36232"
ts=2026-05-12T21:46:51.926959293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:33544"
level=debug ts=2026-05-12T21:46:52.010060142Z caller=checkpoint.go:372 msg="writing series" size="320 kB"
level=debug ts=2026-05-12T21:46:52.010090609Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=11
level=warn ts=2026-05-12T21:46:54.42817377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:46:56.364374393Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-12T21:46:59.427799994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:47:00.718747126Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=warn ts=2026-05-12T21:47:04.428257455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:47:05.07372527Z caller=checkpoint.go:372 msg="writing series" size="282 kB"
level=debug ts=2026-05-12T21:47:09.428193799Z caller=checkpoint.go:384 msg="flushing series" totalSize="295 kB" series=3
level=info ts=2026-05-12T21:47:09.42821141Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:47:09.42823638Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:47:09.428246741Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:47:09.42829382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:47:09.583407695Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-12T21:47:09.58518841Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.157025481s
level=warn ts=2026-05-12T21:47:14.427265194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:47:15.077529275Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T21:47:15.077576278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:47:15.089136771Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T21:47:19.427926104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:47:19.889881608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=info ts=2026-05-12T21:47:21.623839117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:47:21.631445674Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-12T21:47:21.927971342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:44226"
level=warn ts=2026-05-12T21:47:24.428352567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:47:24.979478939Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T21:47:25.7911238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.9 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:47:25.799330754Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T21:47:29.427318178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:47:34.427777996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:47:39.410824791Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:47:39.413539409Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=warn ts=2026-05-12T21:47:39.427792546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:47:39.42780417Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-12T21:47:39.427818619Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:47:39.427837777Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:47:39.427845446Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T21:47:39.427902571Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-12T21:47:39.427909868Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T21:47:39.427926184Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.469µs success=true
level=debug ts=2026-05-12T21:47:39.428178569Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=debug ts=2026-05-12T21:47:43.028351715Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-05-12T21:47:44.427429624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:47:46.628673133Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-12T21:47:49.427854188Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:47:49.78884971Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-12T21:47:49.893579666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=debug ts=2026-05-12T21:47:50.228327797Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
ts=2026-05-12T21:47:51.92889171Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:37656"
level=debug ts=2026-05-12T21:47:53.829421862Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=info ts=2026-05-12T21:47:54.121120683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:47:54.130766027Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T21:47:54.427243522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:47:57.428734506Z caller=checkpoint.go:372 msg="writing series" size="320 kB"
level=warn ts=2026-05-12T21:47:59.427695029Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:48:01.028917198Z caller=checkpoint.go:372 msg="writing series" size="356 kB"
level=warn ts=2026-05-12T21:48:04.427263431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:48:04.628143071Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=debug ts=2026-05-12T21:48:08.228940483Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=info ts=2026-05-12T21:48:09.427725386Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:48:09.42774923Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:48:09.42775676Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:48:09.427807945Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:48:09.738946266Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T21:48:11.130212966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:48:11.136672706Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:48:11.136696886Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T21:48:11.136717916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:48:11.142651058Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:48:11.828957241Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=debug ts=2026-05-12T21:48:11.828982852Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=10
level=warn ts=2026-05-12T21:48:14.428109532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:48:15.08086677Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:48:15.429368852Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=debug ts=2026-05-12T21:48:19.02877258Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-12T21:48:19.427362973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:48:19.895568167Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=info ts=2026-05-12T21:48:22.538348768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:48:22.552769425Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:48:22.628717367Z caller=checkpoint.go:372 msg="writing series" size="305 kB"
level=warn ts=2026-05-12T21:48:24.427623605Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:48:26.228679735Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
ts=2026-05-12T21:48:27.877863333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:58260"
level=warn ts=2026-05-12T21:48:29.427753568Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:48:29.837576762Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-12T21:48:29.837719197Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=5
level=debug ts=2026-05-12T21:48:33.428763214Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-12T21:48:34.427596717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:48:37.02879517Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=info ts=2026-05-12T21:48:39.411490225Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:48:39.4125336Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T21:48:39.427808065Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:48:39.427826118Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:48:39.427834102Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:48:39.427841783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:48:40.369095476Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:48:40.629510644Z caller=checkpoint.go:372 msg="writing series" size="330 kB"
level=info ts=2026-05-12T21:48:41.318328348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:48:41.325691067Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:48:44.228700007Z caller=checkpoint.go:372 msg="writing series" size="9.2 kB"
level=warn ts=2026-05-12T21:48:44.427858055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:48:47.828879149Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-12T21:48:49.427985129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:48:49.898386539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T21:48:50.692352514Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T21:48:51.428539275Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-12T21:48:54.235279965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:48:54.244370978Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:48:54.427565661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:48:55.033039134Z caller=checkpoint.go:372 msg="writing series" size="8.9 MB"
level=debug ts=2026-05-12T21:48:55.033063069Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.4 MB" series=7
level=debug ts=2026-05-12T21:48:58.62843266Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=warn ts=2026-05-12T21:48:59.427843758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:49:02.228559076Z caller=checkpoint.go:372 msg="writing series" size="235 kB"
level=warn ts=2026-05-12T21:49:04.428174706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:49:05.829184378Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T21:49:05.829214704Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-05-12T21:49:09.427790546Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:49:09.427804095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:49:09.427820227Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:49:09.427828497Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:49:09.428900324Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=debug ts=2026-05-12T21:49:13.029439876Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-12T21:49:14.427549088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:49:15.270978591Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:49:16.629194247Z caller=checkpoint.go:372 msg="writing series" size="291 kB"
level=warn ts=2026-05-12T21:49:19.428106673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:49:19.901925368Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T21:49:20.228123648Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
ts=2026-05-12T21:49:21.937618228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:35042"
level=debug ts=2026-05-12T21:49:23.82817524Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=warn ts=2026-05-12T21:49:24.427251682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:49:27.428171542Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-12T21:49:29.427421406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:49:29.799034267Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T21:49:29.800853368Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-12T21:49:31.032439458Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-12T21:49:34.427494435Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:49:34.628203639Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=info ts=2026-05-12T21:49:37.994944977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:49:38.003656861Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:49:38.228399192Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-12T21:49:39.410842339Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:49:39.411058301Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T21:49:39.427885788Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:49:39.427908919Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:49:39.427918408Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:49:39.427930799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:49:40.336351349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:49:40.343342322Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:49:41.793185998Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:49:41.828764559Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-12T21:49:44.427431595Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:49:45.428669402Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=debug ts=2026-05-12T21:49:49.028842277Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-12T21:49:49.428107654Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:49:49.903754356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T21:49:50.962474711Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:44832"
level=debug ts=2026-05-12T21:49:52.628903719Z caller=checkpoint.go:372 msg="writing series" size="308 kB"
level=debug ts=2026-05-12T21:49:52.628932048Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=13
level=warn ts=2026-05-12T21:49:54.428003374Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:49:56.23328656Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
ts=2026-05-12T21:49:57.887514742Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:56044"
level=warn ts=2026-05-12T21:49:59.428212557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:49:59.829046052Z caller=checkpoint.go:372 msg="writing series" size="514 B"
level=debug ts=2026-05-12T21:50:03.428807446Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=warn ts=2026-05-12T21:50:04.427558616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:50:07.02879299Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=warn ts=2026-05-12T21:50:09.4281396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:50:09.42814701Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:50:09.428173127Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:50:09.428183614Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:50:10.628885085Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=debug ts=2026-05-12T21:50:11.228542519Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:50:14.228451517Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=warn ts=2026-05-12T21:50:14.428055682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:50:17.828569506Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=warn ts=2026-05-12T21:50:19.427781942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:50:19.906661095Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T21:50:20.965618966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:51542"
level=debug ts=2026-05-12T21:50:21.428375582Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
ts=2026-05-12T21:50:21.942349026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:45882"
level=debug ts=2026-05-12T21:50:23.345577611Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T21:50:24.42755098Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:50:25.028573094Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
ts=2026-05-12T21:50:27.890740811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:45514"
level=debug ts=2026-05-12T21:50:28.628544448Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-12T21:50:29.428191412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:50:32.22898838Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=info ts=2026-05-12T21:50:33.765434896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:50:33.773820176Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T21:50:34.427427115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:50:35.828909659Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=info ts=2026-05-12T21:50:36.800559043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:50:36.809126742Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T21:50:39.41105034Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:50:39.411539708Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T21:50:39.427913711Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:50:39.427932484Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:50:39.427940449Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:50:39.427954973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:50:39.429120187Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
level=debug ts=2026-05-12T21:50:39.429129194Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=13
level=debug ts=2026-05-12T21:50:43.028711637Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=warn ts=2026-05-12T21:50:44.427883597Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:50:46.628967078Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=debug ts=2026-05-12T21:50:46.864740181Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:50:49.427199009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:50:49.910038105Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T21:50:50.229154007Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
level=debug ts=2026-05-12T21:50:53.828384478Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-05-12T21:50:54.428191251Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:50:57.42849155Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
ts=2026-05-12T21:50:57.894239571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:33774"
level=warn ts=2026-05-12T21:50:59.427850686Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:51:01.028389407Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=warn ts=2026-05-12T21:51:04.428239149Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:51:04.628418068Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-05-12T21:51:08.2285821Z caller=checkpoint.go:372 msg="writing series" size="378 kB"
level=debug ts=2026-05-12T21:51:08.228608289Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=8
level=info ts=2026-05-12T21:51:09.427691534Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:51:09.427696418Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:51:09.427721797Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:51:09.427729492Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:51:11.828817727Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-12T21:51:12.263930132Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T21:51:14.338682856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="117 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:51:14.347589504Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T21:51:14.427594396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:51:15.428328766Z caller=checkpoint.go:372 msg="writing series" size="291 kB"
level=debug ts=2026-05-12T21:51:19.028884691Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-12T21:51:19.427758738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:51:19.913081419Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T21:51:20.943513222Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:51:20.971995868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:58360"
level=debug ts=2026-05-12T21:51:22.62923797Z caller=checkpoint.go:372 msg="writing series" size="243 kB"
level=warn ts=2026-05-12T21:51:24.427686917Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:51:26.228458016Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=warn ts=2026-05-12T21:51:29.42803809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:51:29.731027128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:51:29.738836383Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:51:29.828608092Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=debug ts=2026-05-12T21:51:33.428373171Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-12T21:51:34.42741963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:51:37.028759867Z caller=checkpoint.go:372 msg="writing series" size="371 kB"
level=debug ts=2026-05-12T21:51:37.028790112Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-05-12T21:51:39.410873951Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:51:39.411863067Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T21:51:39.427165651Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:51:39.42718417Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:51:39.427191433Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:51:39.427206418Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:51:39.977891262Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:51:40.629079457Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=debug ts=2026-05-12T21:51:44.22904625Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-12T21:51:44.428128138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:51:47.828803568Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=warn ts=2026-05-12T21:51:49.427777586Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:51:49.915943159Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=debug ts=2026-05-12T21:51:51.428446746Z caller=checkpoint.go:372 msg="writing series" size="176 kB"
ts=2026-05-12T21:51:51.951021091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:59672"
level=warn ts=2026-05-12T21:51:54.427546641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:51:54.459710462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="117 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:51:54.470445908Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:51:55.028752891Z caller=checkpoint.go:372 msg="writing series" size="484 kB"
level=debug ts=2026-05-12T21:51:58.630514902Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-12T21:51:59.427777306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:52:02.228173728Z caller=checkpoint.go:372 msg="writing series" size="6.5 kB"
level=warn ts=2026-05-12T21:52:04.42783023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:52:05.830555949Z caller=checkpoint.go:372 msg="writing series" size="282 kB"
level=debug ts=2026-05-12T21:52:05.830583889Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=debug ts=2026-05-12T21:52:08.410704687Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:52:09.427566695Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:52:09.42760681Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:52:09.427627341Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:52:09.427637815Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T21:52:09.489631154Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-12T21:52:09.494710652Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.066662372s
level=warn ts=2026-05-12T21:52:14.427825114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:52:19.427778963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:52:19.570239882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:52:19.579461252Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-12T21:52:19.919050973Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T21:52:20.978558288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:36888"
level=debug ts=2026-05-12T21:52:22.738209193Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T21:52:24.428026739Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:52:29.427460601Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:52:34.427548843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:52:35.048120922Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T21:52:38.211924064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="117 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:52:38.221455481Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T21:52:39.411342043Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:52:39.411942547Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T21:52:39.427242548Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-12T21:52:39.427251068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:52:39.427261454Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:52:39.427294707Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:52:39.427326003Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T21:52:39.427423121Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-12T21:52:39.427432653Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T21:52:39.42745095Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.675µs success=true
level=debug ts=2026-05-12T21:52:39.427658238Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=debug ts=2026-05-12T21:52:42.934453906Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-05-12T21:52:44.427278433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:52:46.441353706Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-12T21:52:49.427226007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:52:49.922219571Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T21:52:49.947922511Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=debug ts=2026-05-12T21:52:53.454687243Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-12T21:52:54.42779505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:52:56.961090209Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=warn ts=2026-05-12T21:52:59.427955236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:53:00.467469818Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
level=debug ts=2026-05-12T21:53:03.973779703Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=warn ts=2026-05-12T21:53:04.427574926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:53:05.32443082Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:53:07.480333823Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=info ts=2026-05-12T21:53:09.427532033Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:53:09.427556965Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:53:09.427564357Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:53:09.427597808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:53:10.98733758Z caller=checkpoint.go:372 msg="writing series" size="455 kB"
level=debug ts=2026-05-12T21:53:10.987357372Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=10
level=warn ts=2026-05-12T21:53:14.427606694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:53:14.49272096Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-12T21:53:16.224171581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:53:16.239966119Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:53:17.999337505Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-12T21:53:19.427862729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:53:19.925262957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=debug ts=2026-05-12T21:53:20.596978899Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T21:53:21.505973222Z caller=checkpoint.go:372 msg="writing series" size="412 kB"
level=info ts=2026-05-12T21:53:22.804206274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="128 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:53:22.811696671Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:53:22.811718953Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T21:53:22.811742313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="128 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:53:22.818549922Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T21:53:24.427382024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:53:25.012782163Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=debug ts=2026-05-12T21:53:28.52905547Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-12T21:53:28.52908313Z caller=checkpoint.go:384 msg="flushing series" totalSize="17 MB" series=5
level=warn ts=2026-05-12T21:53:29.427838262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:53:32.026032649Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-12T21:53:34.427557529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:53:35.532367363Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=debug ts=2026-05-12T21:53:39.038807294Z caller=checkpoint.go:372 msg="writing series" size="588 kB"
level=info ts=2026-05-12T21:53:39.410935402Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:53:39.411866291Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=warn ts=2026-05-12T21:53:39.427205924Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:53:39.427217902Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:53:39.427235277Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:53:39.427242326Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T21:53:40.244744307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="128 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:53:40.252836468Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:53:40.571237459Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:53:42.545207026Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=warn ts=2026-05-12T21:53:44.427344627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:53:46.051894932Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-12T21:53:49.42778979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:53:49.558185975Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
ts=2026-05-12T21:53:49.927936955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T21:53:50.988151493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:45560"
level=debug ts=2026-05-12T21:53:53.067773593Z caller=checkpoint.go:372 msg="writing series" size="9.8 MB"
level=debug ts=2026-05-12T21:53:53.067796083Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=7
level=warn ts=2026-05-12T21:53:54.427615331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:53:56.571093193Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=debug ts=2026-05-12T21:53:56.668945343Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T21:53:56.670689452Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T21:53:59.427768051Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:54:00.077745093Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-12T21:54:01.216619917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:54:01.223665Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:54:03.584234974Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-12T21:54:03.584255206Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
level=warn ts=2026-05-12T21:54:04.427673604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:54:07.090732247Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-05-12T21:54:09.427371452Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:54:09.427401349Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-12T21:54:09.42739761Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:54:09.427409451Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:54:10.597193091Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-12T21:54:13.780606947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="128 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:54:13.790906509Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:54:14.103598387Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=debug ts=2026-05-12T21:54:14.175765402Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:54:14.427919507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:54:17.610492495Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=warn ts=2026-05-12T21:54:19.428156863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:54:19.933593412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T21:54:20.991340269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:45286"
level=debug ts=2026-05-12T21:54:21.11663467Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=warn ts=2026-05-12T21:54:24.42751557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:54:24.622716371Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=debug ts=2026-05-12T21:54:28.129380103Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-12T21:54:29.427339808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:54:31.636466358Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=warn ts=2026-05-12T21:54:34.428207903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:54:35.142394021Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-12T21:54:38.649426471Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-12T21:54:39.411226113Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:54:39.414060813Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T21:54:39.428008503Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:54:39.428034614Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:54:39.428043571Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:54:39.428083013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:54:42.155465734Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=warn ts=2026-05-12T21:54:44.428044275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:54:45.662214397Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=debug ts=2026-05-12T21:54:45.845275963Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T21:54:47.558663811Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:54:49.168369646Z caller=checkpoint.go:372 msg="writing series" size="401 kB"
level=warn ts=2026-05-12T21:54:49.42734643Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:54:49.936241179Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T21:54:51.968614106Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:55406"
level=debug ts=2026-05-12T21:54:52.675170061Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-12T21:54:54.427419364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:54:56.18193757Z caller=checkpoint.go:372 msg="writing series" size="302 kB"
level=debug ts=2026-05-12T21:54:56.181969465Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=15
level=warn ts=2026-05-12T21:54:59.428103906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:54:59.68837595Z caller=checkpoint.go:372 msg="writing series" size="514 B"
level=info ts=2026-05-12T21:55:01.212062867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:55:01.222248775Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:55:01.222269513Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T21:55:01.222292518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:55:01.228545846Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:55:03.194185561Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-12T21:55:04.427647088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:55:05.771174028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="128 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:55:05.778651882Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:55:06.700990283Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=warn ts=2026-05-12T21:55:09.427923042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:55:09.427928965Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:55:09.427950122Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:55:09.427957475Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:55:10.20748812Z caller=checkpoint.go:372 msg="writing series" size="411 kB"
level=debug ts=2026-05-12T21:55:13.714486109Z caller=checkpoint.go:372 msg="writing series" size="671 kB"
level=debug ts=2026-05-12T21:55:13.714507363Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-05-12T21:55:13.888541272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:55:13.898078929Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:55:14.427674885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:55:17.220181058Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-05-12T21:55:18.974758711Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:55:19.427830399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:55:19.938468191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T21:55:20.727081423Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=debug ts=2026-05-12T21:55:24.233772548Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-12T21:55:24.427535397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:55:27.739592169Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-12T21:55:29.427443806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:55:31.25061235Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=warn ts=2026-05-12T21:55:34.428211334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:55:34.753131744Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=debug ts=2026-05-12T21:55:38.259676181Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-12T21:55:39.41156223Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:55:39.412087534Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T21:55:39.427413739Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:55:39.42743479Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-12T21:55:39.427429522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:55:39.427442843Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:55:41.766404023Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=debug ts=2026-05-12T21:55:42.23983169Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T21:55:43.559832359Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:55:44.428000734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:55:45.272871529Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
level=info ts=2026-05-12T21:55:45.876625496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:55:45.884628448Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:55:48.778979839Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
level=info ts=2026-05-12T21:55:49.243458322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="19 MB" avg_comp="2.7 MB" total_uncomp="152 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:55:49.250243292Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T21:55:49.427519285Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:55:49.94047488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=debug ts=2026-05-12T21:55:52.285446376Z caller=checkpoint.go:372 msg="writing series" size="760 kB"
level=debug ts=2026-05-12T21:55:52.285469623Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=11
level=warn ts=2026-05-12T21:55:54.428237513Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:55:55.792010842Z caller=checkpoint.go:372 msg="writing series" size="776 kB"
level=debug ts=2026-05-12T21:55:59.2988448Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=warn ts=2026-05-12T21:55:59.428076247Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:56:02.804619967Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=debug ts=2026-05-12T21:56:02.80464397Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=warn ts=2026-05-12T21:56:04.427993572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:56:06.311215755Z caller=checkpoint.go:372 msg="writing series" size="529 kB"
level=debug ts=2026-05-12T21:56:08.670681307Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T21:56:09.427632788Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:56:09.427642645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:56:09.427659443Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:56:09.427666358Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:56:09.8183736Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=debug ts=2026-05-12T21:56:13.32485316Z caller=checkpoint.go:372 msg="writing series" size="291 kB"
level=warn ts=2026-05-12T21:56:14.428128202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:56:16.830509261Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=warn ts=2026-05-12T21:56:19.427411489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:56:19.942915507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T21:56:20.337033782Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=debug ts=2026-05-12T21:56:20.337056786Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=debug ts=2026-05-12T21:56:21.400219091Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T21:56:23.843624207Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=warn ts=2026-05-12T21:56:24.428241808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:56:27.349779069Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=info ts=2026-05-12T21:56:28.670323469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:56:28.67774495Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:56:29.427595829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:56:30.856519342Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-12T21:56:34.363871396Z caller=checkpoint.go:372 msg="writing series" size="412 kB"
level=warn ts=2026-05-12T21:56:34.427695453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:56:37.869568054Z caller=checkpoint.go:372 msg="writing series" size="397 kB"
level=debug ts=2026-05-12T21:56:38.560485857Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T21:56:39.411591187Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:56:39.41243956Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T21:56:39.427832144Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:56:39.427846561Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:56:39.427862236Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:56:39.427846472Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:56:41.376089922Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-12T21:56:43.330613397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.7 MB" total_uncomp="176 MB" avg_uncomp="22 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:56:43.340987082Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T21:56:44.427889296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:56:44.882912086Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=debug ts=2026-05-12T21:56:48.389771608Z caller=checkpoint.go:372 msg="writing series" size="177 kB"
level=debug ts=2026-05-12T21:56:48.389792286Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=warn ts=2026-05-12T21:56:49.427886127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:56:49.945185644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=debug ts=2026-05-12T21:56:51.895878166Z caller=checkpoint.go:372 msg="writing series" size="726 kB"
level=warn ts=2026-05-12T21:56:54.42783008Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:56:55.402661601Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
ts=2026-05-12T21:56:57.929752043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:43040"
level=debug ts=2026-05-12T21:56:58.908649999Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-12T21:56:59.427507539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:57:02.41474074Z caller=checkpoint.go:372 msg="writing series" size="282 kB"
level=debug ts=2026-05-12T21:57:02.920159419Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T21:57:04.427849075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:57:05.921770461Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-12T21:57:09.427329251Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:57:09.427353993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:57:09.427368915Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:57:09.427379132Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:57:09.428401147Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.0 MB" series=5
level=info ts=2026-05-12T21:57:09.439000708Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-12T21:57:09.440811049Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.013228109s
level=warn ts=2026-05-12T21:57:14.427234622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:57:15.95529078Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T21:57:18.400402602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:57:18.407900227Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:57:19.428001098Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:57:19.948132496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=warn ts=2026-05-12T21:57:24.427351491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:57:29.427932253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T21:57:34.428230392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:57:36.15662084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="22 MB" avg_comp="2.7 MB" total_uncomp="176 MB" avg_uncomp="22 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:57:36.167493842Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T21:57:37.925175777Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T21:57:39.410808319Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:57:39.411225419Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T21:57:39.427702727Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:57:39.427718953Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:57:39.427713053Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-12T21:57:39.427726455Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:57:39.427754942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T21:57:39.42786992Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-12T21:57:39.427878639Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T21:57:39.427898814Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.611µs success=true
level=debug ts=2026-05-12T21:57:39.428157877Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=debug ts=2026-05-12T21:57:42.935448525Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-12T21:57:44.427488909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:57:46.442009763Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-12T21:57:49.427806772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:57:49.948074697Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
ts=2026-05-12T21:57:49.951351526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T21:57:51.014618193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:34592"
level=debug ts=2026-05-12T21:57:52.006370479Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T21:57:52.008244676Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-12T21:57:53.454979571Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-12T21:57:54.427504838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:57:56.960922478Z caller=checkpoint.go:372 msg="writing series" size="360 kB"
ts=2026-05-12T21:57:57.934403204Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:42044"
level=warn ts=2026-05-12T21:57:59.427951545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:58:00.46752958Z caller=checkpoint.go:372 msg="writing series" size="392 kB"
level=debug ts=2026-05-12T21:58:03.974129654Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=warn ts=2026-05-12T21:58:04.427561116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:58:07.480629419Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=debug ts=2026-05-12T21:58:07.480655496Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-05-12T21:58:09.427173381Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:58:09.42721308Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:58:09.427224296Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:58:09.427245405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:58:09.609421008Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:58:10.987369309Z caller=checkpoint.go:372 msg="writing series" size="455 kB"
level=warn ts=2026-05-12T21:58:14.427254775Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:58:14.493598091Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=info ts=2026-05-12T21:58:16.32136082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="24 MB" avg_comp="2.7 MB" total_uncomp="199 MB" avg_uncomp="22 MB" full=8 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:58:16.331686286Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T21:58:17.886142861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:58:17.89486107Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T21:58:18.000264118Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-12T21:58:19.428079842Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:58:19.955680093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T21:58:21.018359471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:53926"
level=debug ts=2026-05-12T21:58:21.506518312Z caller=checkpoint.go:372 msg="writing series" size="323 kB"
ts=2026-05-12T21:58:21.98729483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:37556"
level=warn ts=2026-05-12T21:58:24.427760125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:58:25.012802848Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
ts=2026-05-12T21:58:27.941316125Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:54542"
level=debug ts=2026-05-12T21:58:28.54456505Z caller=checkpoint.go:372 msg="writing series" size="26 MB"
level=debug ts=2026-05-12T21:58:28.544614578Z caller=checkpoint.go:384 msg="flushing series" totalSize="27 MB" series=6
level=warn ts=2026-05-12T21:58:29.427508331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:58:32.025980362Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-12T21:58:34.427795079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:58:35.533141955Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-05-12T21:58:36.07503209Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T21:58:39.039662872Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-12T21:58:39.039688805Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=debug ts=2026-05-12T21:58:39.41074721Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T21:58:39.410992905Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T21:58:39.427488398Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:58:39.427512229Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:58:39.427519966Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:58:39.427540084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:58:42.546041035Z caller=checkpoint.go:372 msg="writing series" size="271 kB"
level=warn ts=2026-05-12T21:58:44.427261157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:58:45.359410089Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:58:46.052452736Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=warn ts=2026-05-12T21:58:49.427913362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:58:49.558332813Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
ts=2026-05-12T21:58:49.964932428Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T21:58:53.073626145Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-12T21:58:53.073647703Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=4
level=warn ts=2026-05-12T21:58:54.427551672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:58:56.571502907Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=info ts=2026-05-12T21:58:58.578946257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:58:58.587540116Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T21:58:59.427969075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:59:00.078504571Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=debug ts=2026-05-12T21:59:03.585803257Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-12T21:59:03.58582437Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=3
level=warn ts=2026-05-12T21:59:04.427506213Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:59:07.09073881Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-12T21:59:07.728738829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="24 MB" avg_comp="2.7 MB" total_uncomp="199 MB" avg_uncomp="22 MB" full=8 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:59:07.737834477Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T21:59:09.427203537Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T21:59:09.427214435Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:59:09.427227406Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:59:09.427242908Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T21:59:10.597716526Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-12T21:59:12.069442208Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T21:59:13.635913836Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T21:59:14.104114097Z caller=checkpoint.go:372 msg="writing series" size="218 kB"
level=warn ts=2026-05-12T21:59:14.427653891Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:59:17.610936744Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=warn ts=2026-05-12T21:59:19.427709996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:59:19.969000681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T21:59:21.117142295Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=warn ts=2026-05-12T21:59:24.42741955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:59:24.623334434Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-12T21:59:28.129798358Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-12T21:59:29.427375088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:59:31.637126474Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=warn ts=2026-05-12T21:59:34.427495151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:59:35.143290801Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-12T21:59:38.649667632Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-05-12T21:59:39.362785822Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T21:59:39.411023511Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T21:59:39.411579081Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T21:59:39.428006138Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T21:59:39.428024784Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T21:59:39.428031688Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T21:59:39.428056615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:59:42.156437019Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=debug ts=2026-05-12T21:59:42.156458245Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=11
level=warn ts=2026-05-12T21:59:44.428049697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:59:45.662938358Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=debug ts=2026-05-12T21:59:49.168961226Z caller=checkpoint.go:372 msg="writing series" size="301 kB"
level=warn ts=2026-05-12T21:59:49.427342042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T21:59:49.971694753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=info ts=2026-05-12T21:59:50.500330294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="27 MB" avg_comp="2.7 MB" total_uncomp="223 MB" avg_uncomp="22 MB" full=9 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:59:50.510153991Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:59:50.510179851Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T21:59:50.510208738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="27 MB" avg_comp="2.7 MB" total_uncomp="223 MB" avg_uncomp="22 MB" full=9 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T21:59:50.516764046Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-12T21:59:51.994389699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:47354"
level=debug ts=2026-05-12T21:59:52.675855434Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=warn ts=2026-05-12T21:59:54.428058396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T21:59:56.181709154Z caller=checkpoint.go:372 msg="writing series" size="590 kB"
level=debug ts=2026-05-12T21:59:56.181730094Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-12T21:59:57.745263213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:59:57.752837666Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T21:59:57.753042236Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=6164 at timestamp 2026-05-12T21:59:43Z for stream {stream=\"build-e2e-hfht\"}" org_id=kubearchive
level=error ts=2026-05-12T21:59:57.753086995Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=198 at timestamp 2026-05-12T21:59:44Z for stream {stream=\"build-e2e-hfht\"}" org_id=kubearchive
level=error ts=2026-05-12T21:59:57.753104729Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=22863 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"openshift-ovn-kubernetes\"}" org_id=kubearchive
level=error ts=2026-05-12T21:59:57.753113744Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=23336 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"openshift-ovn-kubernetes\"}" org_id=kubearchive
level=error ts=2026-05-12T21:59:57.753126142Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T21:59:57.753130841Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T21:59:57.753135558Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T21:59:57.753140792Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T21:59:57.753145061Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
ts=2026-05-12T21:59:57.948230671Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:35328"
level=warn ts=2026-05-12T21:59:59.427657503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-12T21:59:59.526182659Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=159 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"kube-system\"}" org_id=kubearchive
level=error ts=2026-05-12T21:59:59.558521794Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=271 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"openshift-console\"}" org_id=kubearchive
level=error ts=2026-05-12T21:59:59.558551319Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=1209 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"konflux-kite\"}" org_id=kubearchive
level=error ts=2026-05-12T21:59:59.558753115Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=info ts=2026-05-12T22:00:00.8168877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="30 MB" avg_comp="2.7 MB" total_uncomp="244 MB" avg_uncomp="22 MB" full=10 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:00:00.825005057Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:00:01.026517086Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=3378 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:01.026570447Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=738 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:01.026577461Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:01.026583409Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:01.026596745Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=12103 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"openshift-image-registry\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:01.026605602Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=411 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"openshift-cluster-csi-drivers\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:01.026613851Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=268 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:01.02662566Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=777 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:01.026630128Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=21 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=warn ts=2026-05-12T22:00:04.427898818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-12T22:00:04.451749798Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=6164 at timestamp 2026-05-12T21:59:43Z for stream {stream=\"build-e2e-hfht\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:04.451801075Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=198 at timestamp 2026-05-12T21:59:44Z for stream {stream=\"build-e2e-hfht\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:04.451824395Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:04.451829552Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:04.451834592Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:04.451840702Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:04.451847603Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:04.45185429Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:04.451858315Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=65 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:04.851268611Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=1022 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"openshift-cluster-csi-drivers\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:04.851333551Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=271 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"openshift-console\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:04.852457419Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:52Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:04.852470397Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:05.013963757Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=159 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"kube-system\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:05.01538291Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=271 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"openshift-console\"}" org_id=kubearchive
level=debug ts=2026-05-12T22:00:06.175417887Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-12T22:00:06.487097379Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=3378 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:06.487148563Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=738 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:06.487156994Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:06.487162593Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:06.487175401Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=12103 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"openshift-image-registry\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:06.487184627Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=411 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"openshift-cluster-csi-drivers\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:06.487193454Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=268 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:06.487205424Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=777 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:06.487210095Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=21 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=debug ts=2026-05-12T22:00:08.50487667Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:00:09.427946919Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:00:09.427975287Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:00:09.427982474Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:00:09.42803202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-12T22:00:09.932626797Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=1022 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"openshift-cluster-csi-drivers\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:09.932687211Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=271 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"openshift-console\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:09.933735612Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:52Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:09.933750092Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:09.933755979Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:09.93376172Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=286 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:09.933770659Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=5182 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"openshift-image-registry\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:09.934617814Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=579 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:09.934629319Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=579 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:11.571459779Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=6164 at timestamp 2026-05-12T21:59:43Z for stream {stream=\"build-e2e-hfht\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:11.571514473Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=198 at timestamp 2026-05-12T21:59:44Z for stream {stream=\"build-e2e-hfht\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:11.571801637Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:11.571809536Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:11.571814897Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:11.571820734Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:11.571825017Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:11.571829628Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:11.571834081Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=65 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:11.893067901Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=159 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"kube-system\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:11.894435555Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=271 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"openshift-console\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:11.894453737Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=1209 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"konflux-kite\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:11.894645493Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:13.086509295Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=3378 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:13.086560098Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=738 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:13.086567711Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:13.086573417Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:13.086586103Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=12103 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"openshift-image-registry\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:13.086596937Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=411 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"openshift-cluster-csi-drivers\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:13.086605562Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=268 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:13.086618435Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=777 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:13.086623202Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=21 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=warn ts=2026-05-12T22:00:14.427747243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-12T22:00:17.103559404Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=159 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"kube-system\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:17.105258079Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:17.105286119Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=22471 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"openshift-ovn-kubernetes\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:17.105294077Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=22783 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"openshift-ovn-kubernetes\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:17.105324965Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=22634 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"openshift-ovn-kubernetes\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:17.105336239Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=22546 at timestamp 2026-05-12T21:59:52Z for stream {stream=\"openshift-ovn-kubernetes\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:17.640316863Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=31935 at timestamp 2026-05-12T21:59:55Z for stream {stream=\"openshift-gitops\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:17.640396764Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=1022 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"openshift-cluster-csi-drivers\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:17.640418407Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=271 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"openshift-console\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:17.641707225Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:52Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:17.641727283Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:17.641735467Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:17.641744961Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=286 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:17.641761294Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=5182 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"openshift-image-registry\"}" org_id=kubearchive
level=warn ts=2026-05-12T22:00:19.427546744Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:00:19.974983714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=error ts=2026-05-12T22:00:22.088719607Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=3378 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:22.088768437Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=738 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:22.088774974Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:22.088780783Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:22.088792668Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=12103 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"openshift-image-registry\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:22.08880223Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=411 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"openshift-cluster-csi-drivers\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:22.088812416Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=268 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:22.088826051Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=777 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:22.08883136Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=21 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:23.802054668Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=31935 at timestamp 2026-05-12T21:59:55Z for stream {stream=\"openshift-gitops\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:23.802105755Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=1022 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"openshift-cluster-csi-drivers\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:23.803048061Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:52Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:23.803060924Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:23.803065994Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:23.803070777Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=286 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:23.803079498Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=5182 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"openshift-image-registry\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:23.80386324Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=579 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:23.803871381Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=579 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:23.958208005Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=6164 at timestamp 2026-05-12T21:59:43Z for stream {stream=\"build-e2e-hfht\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:23.958261842Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=198 at timestamp 2026-05-12T21:59:44Z for stream {stream=\"build-e2e-hfht\"}" org_id=kubearchive
level=warn ts=2026-05-12T22:00:24.427648527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:00:27.951489775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:54602"
level=error ts=2026-05-12T22:00:29.309281999Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=159 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"kube-system\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:29.333221264Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=271 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"openshift-console\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:29.333245831Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=1209 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"konflux-kite\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:29.333469734Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=warn ts=2026-05-12T22:00:29.427528452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:00:32.483099628Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:00:32.97134589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="31 MB" avg_comp="2.6 MB" total_uncomp="253 MB" avg_uncomp="21 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:00:32.979592366Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:00:33.246359724Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=31935 at timestamp 2026-05-12T21:59:55Z for stream {stream=\"openshift-gitops\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:33.246427138Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=271 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"openshift-console\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:33.247587298Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:52Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:33.247606337Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:33.247614895Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:33.247622616Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=286 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:33.24878408Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=579 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:33.248793362Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=579 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:33.248798702Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=18 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:33.275040037Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=3378 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=warn ts=2026-05-12T22:00:34.427586393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-12T22:00:38.59336287Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=3378 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:38.593418152Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=738 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:38.59342556Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:38.593431928Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:38.593454926Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=777 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:38.593459703Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=21 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:38.593464653Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=828 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:38.593474292Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=2860 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"kube-system\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:38.593481582Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=18403 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"kube-system\"}" org_id=kubearchive
level=info ts=2026-05-12T22:00:39.410993485Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:00:39.411241683Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=debug ts=2026-05-12T22:00:39.412479167Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-12T22:00:39.412519263Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976247 n=1
level=debug ts=2026-05-12T22:00:39.412575965Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976247 n=1
level=info ts=2026-05-12T22:00:39.427703712Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:00:39.427724205Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:00:39.427732022Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:00:39.427744072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-12T22:00:39.524382143Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=6164 at timestamp 2026-05-12T21:59:43Z for stream {stream=\"build-e2e-hfht\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:39.524431955Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=198 at timestamp 2026-05-12T21:59:44Z for stream {stream=\"build-e2e-hfht\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:39.524454087Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:39.524459815Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:39.524465001Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:39.524470952Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:39.524475409Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:39.524480521Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:39.52448497Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=65 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=warn ts=2026-05-12T22:00:44.427534682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-12T22:00:44.655243065Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=159 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"kube-system\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:44.656921044Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=271 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"openshift-console\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:44.656941576Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=1209 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"konflux-kite\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:44.657153232Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=debug ts=2026-05-12T22:00:47.974617237Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:00:48.803434669Z caller=checkpoint.go:372 msg="writing series" size="514 B"
level=error ts=2026-05-12T22:00:49.234493132Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=31935 at timestamp 2026-05-12T21:59:55Z for stream {stream=\"openshift-gitops\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:49.234548305Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=1022 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"openshift-cluster-csi-drivers\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:49.234574563Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=271 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"openshift-console\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:49.235421602Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:52Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:49.235434448Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:49.235441592Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:49.23545017Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=286 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:49.235462707Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=5182 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"openshift-image-registry\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:49.236265642Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=579 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=warn ts=2026-05-12T22:00:49.427229221Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-12T22:00:49.519745452Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=3378 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:49.519790534Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=738 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:49.519797332Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
ts=2026-05-12T22:00:49.978279071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:00:52.285803356Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=warn ts=2026-05-12T22:00:54.427942845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-12T22:00:54.484210463Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=159 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"kube-system\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:54.485712685Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=271 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"openshift-console\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:54.485742709Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=info ts=2026-05-12T22:00:55.536514946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:00:55.547989353Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:00:55.792874696Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=debug ts=2026-05-12T22:00:59.026943586Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:00:59.298874301Z caller=checkpoint.go:372 msg="writing series" size="583 kB"
level=warn ts=2026-05-12T22:00:59.427790252Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-12T22:00:59.834657547Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=20928 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"openshift-ovn-kubernetes\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:59.834723594Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=1022 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"openshift-cluster-csi-drivers\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:59.834740804Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=49522 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"product-kubearchive\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:59.834750048Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=271 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"openshift-console\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:59.835812628Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:52Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:59.835831972Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:59.835839094Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:59.83584687Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=286 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:00:59.835858583Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=5182 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"openshift-image-registry\"}" org_id=kubearchive
level=debug ts=2026-05-12T22:01:02.80589824Z caller=checkpoint.go:372 msg="writing series" size="807 kB"
level=debug ts=2026-05-12T22:01:02.805920355Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=error ts=2026-05-12T22:01:02.994405508Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=6164 at timestamp 2026-05-12T21:59:43Z for stream {stream=\"build-e2e-hfht\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:02.994456104Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=198 at timestamp 2026-05-12T21:59:44Z for stream {stream=\"build-e2e-hfht\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:02.99475513Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:02.994762749Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:02.994767848Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:02.994772508Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:02.994776637Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:02.994780786Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:02.994784956Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=65 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=warn ts=2026-05-12T22:01:04.42814065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-12T22:01:08.300628808Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=6164 at timestamp 2026-05-12T21:59:43Z for stream {stream=\"build-e2e-hfht\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:08.300683965Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=198 at timestamp 2026-05-12T21:59:44Z for stream {stream=\"build-e2e-hfht\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:08.300708653Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:08.300714278Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:08.300719844Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:08.300724647Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:08.30072889Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:08.30073335Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:08.300737784Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=65 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=info ts=2026-05-12T22:01:09.428037841Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:01:09.428064723Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:01:09.428072491Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:01:09.428089903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-12T22:01:12.578967129Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=159 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"kube-system\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:12.581240973Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=271 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"openshift-console\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:12.581262858Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=1209 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"konflux-kite\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:12.581279044Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=warn ts=2026-05-12T22:01:14.427648765Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:01:14.938709733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="31 MB" avg_comp="2.6 MB" total_uncomp="253 MB" avg_uncomp="21 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:01:14.947896859Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:01:17.453055637Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=3378 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:17.453104044Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=738 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:17.453111713Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:17.453117129Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:17.453129425Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=12103 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"openshift-image-registry\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:17.453140281Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=411 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"openshift-cluster-csi-drivers\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:17.453150164Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=268 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:17.453162388Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=777 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:17.453168168Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=21 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:18.890445779Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=31935 at timestamp 2026-05-12T21:59:55Z for stream {stream=\"openshift-gitops\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:18.890471836Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=1022 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"openshift-cluster-csi-drivers\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:18.890755685Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=271 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"openshift-console\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:18.891645214Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:52Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:18.891656638Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:18.891662628Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:18.891668168Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=286 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:18.891676813Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=5182 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"openshift-image-registry\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:18.892587771Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=579 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=warn ts=2026-05-12T22:01:19.42807394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:01:19.98045206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=warn ts=2026-05-12T22:01:24.427256332Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-12T22:01:25.846655061Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:52Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:25.846677419Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:25.846683381Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:25.846689261Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=286 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:25.952725582Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=579 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:25.952748304Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=579 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:25.952754752Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=18 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:25.95276109Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=5131 at timestamp 2026-05-12T21:59:55Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:25.9527671Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=6812 at timestamp 2026-05-12T21:59:56Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:25.952772439Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=1268 at timestamp 2026-05-12T21:59:56Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=warn ts=2026-05-12T22:01:29.427368635Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-12T22:01:32.413177093Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=6164 at timestamp 2026-05-12T21:59:43Z for stream {stream=\"build-e2e-hfht\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:32.413239154Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=198 at timestamp 2026-05-12T21:59:44Z for stream {stream=\"build-e2e-hfht\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:32.413266978Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:32.413272629Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:32.413277736Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:32.413283342Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:32.41328784Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:32.413292365Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:32.413313455Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=65 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=debug ts=2026-05-12T22:01:32.497001381Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:01:34.427516556Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:01:37.843572376Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:01:37.846263163Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-12T22:01:39.410980368Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:01:39.411379623Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:01:39.427847752Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:01:39.427852346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:01:39.427866933Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:01:39.427874213Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=error ts=2026-05-12T22:01:40.466798411Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=159 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"kube-system\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:40.468650207Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=271 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"openshift-console\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:40.468676151Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=1209 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"konflux-kite\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:40.468696931Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:41.63495296Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=3378 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:41.635016513Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=738 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:41.635033702Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:41.635045087Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:41.6350637Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=12103 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"openshift-image-registry\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:41.635127565Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=411 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"openshift-cluster-csi-drivers\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:41.635144385Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=268 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:41.635163887Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=777 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:41.635170429Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=21 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=warn ts=2026-05-12T22:01:44.427580302Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-12T22:01:46.90625594Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=6164 at timestamp 2026-05-12T21:59:43Z for stream {stream=\"build-e2e-hfht\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:46.906336163Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=198 at timestamp 2026-05-12T21:59:44Z for stream {stream=\"build-e2e-hfht\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:46.906357068Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=25542 at timestamp 2026-05-12T21:59:45Z for stream {stream=\"openshift-ovn-kubernetes\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:46.906366987Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=22590 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"openshift-ovn-kubernetes\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:46.906375285Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=22863 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"openshift-ovn-kubernetes\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:46.906386345Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=23336 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"openshift-ovn-kubernetes\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:46.906396779Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:46.90640492Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:46.906409792Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:47.687553969Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=159 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"kube-system\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:47.700194209Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=1209 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"konflux-kite\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:47.700263951Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=41080 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"product-kubearchive\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:47.700282603Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=41259 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"product-kubearchive\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:47.70033478Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=41158 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"product-kubearchive\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:47.700351087Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=39955 at timestamp 2026-05-12T21:59:52Z for stream {stream=\"product-kubearchive\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:47.700366589Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=warn ts=2026-05-12T22:01:49.427349828Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:01:49.982451513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=info ts=2026-05-12T22:01:50.353376699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:01:50.38073252Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:01:50.380764909Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:01:50.380804535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:01:50.398897402Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-12T22:01:51.035329499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:39518"
level=warn ts=2026-05-12T22:01:54.427469325Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-12T22:01:55.504072283Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=3378 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:55.504157363Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=738 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:55.504169459Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:55.504186201Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:55.504209478Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=268 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:55.504232873Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=777 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:55.504243198Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=21 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:55.504253443Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=828 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:55.504271799Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=2860 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"kube-system\"}" org_id=kubearchive
level=info ts=2026-05-12T22:01:56.50624298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="32 MB" avg_comp="2.5 MB" total_uncomp="262 MB" avg_uncomp="20 MB" full=12 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:01:56.530019424Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:01:57.362539353Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=159 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"kube-system\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:57.376401254Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=41080 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"product-kubearchive\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:57.37947807Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=41259 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"product-kubearchive\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:57.379512301Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=41158 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"product-kubearchive\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:57.379536637Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=39955 at timestamp 2026-05-12T21:59:52Z for stream {stream=\"product-kubearchive\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:57.379557503Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:57.379649777Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=22471 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"openshift-ovn-kubernetes\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:57.379678606Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=22783 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"openshift-ovn-kubernetes\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:57.379701872Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=22634 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"openshift-ovn-kubernetes\"}" org_id=kubearchive
level=debug ts=2026-05-12T22:01:57.508185596Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
ts=2026-05-12T22:01:57.96019855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:42504"
level=debug ts=2026-05-12T22:01:58.104103945Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:01:58.909381294Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=error ts=2026-05-12T22:01:59.097398326Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=31935 at timestamp 2026-05-12T21:59:55Z for stream {stream=\"openshift-gitops\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:59.097444388Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=1022 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"openshift-cluster-csi-drivers\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:59.099350702Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=271 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"openshift-console\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:59.100513796Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:52Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:59.10052832Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:59.100533656Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:59.100539027Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=286 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:59.100547411Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=5182 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"openshift-image-registry\"}" org_id=kubearchive
level=error ts=2026-05-12T22:01:59.101130306Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=579 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=warn ts=2026-05-12T22:01:59.427460377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:02:02.416037344Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-12T22:02:04.427419004Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:02:05.922564546Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=error ts=2026-05-12T22:02:07.024211497Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=3378 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:07.024261705Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=738 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:07.024268195Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:07.024274353Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:07.024286386Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=12103 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"openshift-image-registry\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:07.024324554Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=411 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"openshift-cluster-csi-drivers\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:07.024335162Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=268 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:07.024348709Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=777 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:07.02435354Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=21 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=info ts=2026-05-12T22:02:07.509876254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:02:07.521567168Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:02:09.427397503Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:02:09.427435576Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:02:09.42744311Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:02:09.427443189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:02:09.428650842Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=error ts=2026-05-12T22:02:11.020711836Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=6164 at timestamp 2026-05-12T21:59:43Z for stream {stream=\"build-e2e-hfht\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:11.020768884Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=198 at timestamp 2026-05-12T21:59:44Z for stream {stream=\"build-e2e-hfht\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:11.020796618Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:11.02080397Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:11.020809319Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:11.020814258Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:11.02081856Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:11.020823921Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:11.020828173Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=65 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=debug ts=2026-05-12T22:02:12.934768373Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=error ts=2026-05-12T22:02:14.024281432Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=159 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"kube-system\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:14.026338777Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=271 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"openshift-console\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:14.02636118Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=1209 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"konflux-kite\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:14.026378638Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=warn ts=2026-05-12T22:02:14.428049729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:02:16.441464683Z caller=checkpoint.go:372 msg="writing series" size="151 kB"
level=warn ts=2026-05-12T22:02:19.428165968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:02:19.947861915Z caller=checkpoint.go:372 msg="writing series" size="83 kB"
ts=2026-05-12T22:02:19.98803277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T22:02:22.007869092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:46798"
level=debug ts=2026-05-12T22:02:22.415497789Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:02:23.454388568Z caller=checkpoint.go:372 msg="writing series" size="225 kB"
level=error ts=2026-05-12T22:02:23.535869284Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=3378 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:23.535917239Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=738 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:23.535924203Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:23.5359302Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:23.535949162Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=268 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:23.535964424Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=777 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:23.53597152Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=21 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:23.535978144Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=828 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:23.535987131Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=2860 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"kube-system\"}" org_id=kubearchive
level=warn ts=2026-05-12T22:02:24.428350408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-12T22:02:25.751764659Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=31935 at timestamp 2026-05-12T21:59:55Z for stream {stream=\"openshift-gitops\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:25.751792778Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=1022 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"openshift-cluster-csi-drivers\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:25.752366335Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=271 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"openshift-console\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:25.753281419Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:52Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:25.753292019Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:25.753312589Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:25.753317803Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=286 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:25.753326291Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=5182 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"openshift-image-registry\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:25.754312224Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=579 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=debug ts=2026-05-12T22:02:26.877615698Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:02:26.960971034Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=warn ts=2026-05-12T22:02:29.42776365Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-12T22:02:29.728613251Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=6164 at timestamp 2026-05-12T21:59:43Z for stream {stream=\"build-e2e-hfht\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:29.728658399Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=198 at timestamp 2026-05-12T21:59:44Z for stream {stream=\"build-e2e-hfht\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:29.728678709Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:29.728684327Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:29.728691541Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:29.728700762Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:29.728705338Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:29.728710358Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:29.728716369Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=65 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=debug ts=2026-05-12T22:02:30.470148397Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-12T22:02:30.470180064Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=11
level=error ts=2026-05-12T22:02:30.885337051Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=31935 at timestamp 2026-05-12T21:59:55Z for stream {stream=\"openshift-gitops\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:30.890692048Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:52Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:30.890721107Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:30.890729942Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:30.89077273Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=286 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:31.083683669Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=579 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:31.083707397Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=579 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:31.083719184Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=18 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:31.083725922Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=5131 at timestamp 2026-05-12T21:59:55Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:31.083732764Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=6812 at timestamp 2026-05-12T21:59:56Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:31.083741995Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=1268 at timestamp 2026-05-12T21:59:56Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=warn ts=2026-05-12T22:02:34.427462438Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:02:37.782786579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:02:37.804800362Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:02:38.135937263Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:52Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:38.13596564Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:38.135976535Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:38.135995796Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=286 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:38.140198306Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=579 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:38.140225228Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=579 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:38.140233096Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=18 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:38.140241241Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=5131 at timestamp 2026-05-12T21:59:55Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:38.140256759Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=6812 at timestamp 2026-05-12T21:59:56Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=info ts=2026-05-12T22:02:39.411804643Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:02:39.413669972Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:02:39.427244532Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:02:39.427275933Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:02:39.427286257Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:02:39.427468553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:02:43.782707271Z caller=checkpoint.go:372 msg="writing series" size="949 kB"
level=debug ts=2026-05-12T22:02:43.782934784Z caller=checkpoint.go:372 msg="writing series" size="149 kB"
level=debug ts=2026-05-12T22:02:43.782962115Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=warn ts=2026-05-12T22:02:44.427596323Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:02:44.493976621Z caller=checkpoint.go:372 msg="writing series" size="533 kB"
level=error ts=2026-05-12T22:02:47.074703052Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=159 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"kube-system\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:47.077703419Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=271 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"openshift-console\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:47.077735775Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=1209 at timestamp 2026-05-12T21:59:49Z for stream {stream=\"konflux-kite\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:47.077923477Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"build-e2e-hevv\"}" org_id=kubearchive
level=debug ts=2026-05-12T22:02:48.000172119Z caller=checkpoint.go:372 msg="writing series" size="547 kB"
level=debug ts=2026-05-12T22:02:48.000197037Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=error ts=2026-05-12T22:02:48.743431277Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=3378 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:48.743496668Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=738 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:48.743508119Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:48.743517824Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:51Z for stream {stream=\"konflux-bflt-managed\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:48.743534651Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=12103 at timestamp 2026-05-12T21:59:46Z for stream {stream=\"openshift-image-registry\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:48.743555094Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=411 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"openshift-cluster-csi-drivers\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:48.743576883Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=8191 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"appstudio-registry-exporter\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:48.743591947Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=777 at timestamp 2026-05-12T21:59:47Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:48.74360078Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=21 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=warn ts=2026-05-12T22:02:49.42796193Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:02:49.529586247Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-12T22:02:49.990995959Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=info ts=2026-05-12T22:02:50.887355626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="33 MB" avg_comp="2.4 MB" total_uncomp="271 MB" avg_uncomp="19 MB" full=13 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:02:50.898241195Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:02:51.506623204Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=warn ts=2026-05-12T22:02:54.42810404Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:02:55.013145898Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=debug ts=2026-05-12T22:02:58.519291224Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=warn ts=2026-05-12T22:02:59.42721219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-12T22:02:59.838641286Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=1022 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"openshift-cluster-csi-drivers\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:59.838717657Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=271 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"openshift-console\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:59.839974492Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:52Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:59.839993306Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=131 at timestamp 2026-05-12T21:59:53Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:59.840002668Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:59.840011033Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=286 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"build-e2e-eolb\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:59.876507127Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=45541 at timestamp 2026-05-12T21:59:54Z for stream {stream=\"integration-service\"}" org_id=kubearchive
level=error ts=2026-05-12T22:02:59.876534674Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=42913 at timestamp 2026-05-12T21:59:57Z for stream {stream=\"integration-service\"}" org_id=kubearchive
level=error ts=2026-05-12T22:03:00.486518347Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=6164 at timestamp 2026-05-12T21:59:43Z for stream {stream=\"build-e2e-hfht\"}" org_id=kubearchive
level=error ts=2026-05-12T22:03:00.486570942Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=198 at timestamp 2026-05-12T21:59:44Z for stream {stream=\"build-e2e-hfht\"}" org_id=kubearchive
level=error ts=2026-05-12T22:03:00.486614427Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=315 at timestamp 2026-05-12T21:59:50Z for stream {stream=\"kube-system\"}" org_id=kubearchive
level=error ts=2026-05-12T22:03:00.486638462Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=118133 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"integration-service\"}" org_id=kubearchive
level=error ts=2026-05-12T22:03:00.487894915Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=483 at timestamp 2026-05-12T21:59:48Z for stream {stream=\"konflux-kite\"}" org_id=kubearchive
level=error ts=2026-05-12T22:03:00.487957895Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=7882 at timestamp 2026-05-12T21:59:43Z for stream {stream=\"build-e2e-muut\"}" org_id=kubearchive
level=error ts=2026-05-12T22:03:00.487965696Z caller=manager.go:49 component=ingester path=write msg="write operation failed" details="duplicate log entry with size=1609 at timestamp 2026-05-12T21:59:44Z for stream {stream=\"build-e2e-muut\"}" org_id=kubearchive
level=debug ts=2026-05-12T22:03:02.025819715Z caller=checkpoint.go:372 msg="writing series" size="301 kB"
level=warn ts=2026-05-12T22:03:04.428174586Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:03:05.532956729Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=debug ts=2026-05-12T22:03:09.039348529Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=info ts=2026-05-12T22:03:09.428131881Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:03:09.428161278Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:03:09.428170506Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:03:09.428191165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:03:12.545732649Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-12T22:03:14.427839714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:03:16.052602135Z caller=checkpoint.go:372 msg="writing series" size="441 kB"
level=debug ts=2026-05-12T22:03:16.052623568Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=warn ts=2026-05-12T22:03:19.427790855Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:03:19.558291986Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
ts=2026-05-12T22:03:19.994561499Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:03:20.878247368Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:03:22.499034556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:03:22.506631981Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:03:23.065171057Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-12T22:03:24.398608254Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:03:24.427615218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:03:26.571385997Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=warn ts=2026-05-12T22:03:29.427801015Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:03:30.077853142Z caller=checkpoint.go:372 msg="writing series" size="178 kB"
level=debug ts=2026-05-12T22:03:33.586337953Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-12T22:03:33.586359033Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=5
level=warn ts=2026-05-12T22:03:34.427635504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:03:37.090712669Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-05-12T22:03:39.411345079Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:03:39.411769007Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:03:39.427197675Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:03:39.427225045Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-12T22:03:39.427223952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:03:39.427235181Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:03:40.597914238Z caller=checkpoint.go:372 msg="writing series" size="9.1 kB"
level=debug ts=2026-05-12T22:03:44.104352614Z caller=checkpoint.go:372 msg="writing series" size="282 kB"
level=warn ts=2026-05-12T22:03:44.427708898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:03:46.299310098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="39 MB" avg_comp="2.4 MB" total_uncomp="318 MB" avg_uncomp="20 MB" full=15 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:03:46.307254986Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:03:47.610742912Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-12T22:03:49.428099147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:03:49.99778679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:03:51.117343214Z caller=checkpoint.go:384 msg="flushing series" totalSize="311 kB" series=4
level=info ts=2026-05-12T22:03:51.119647506Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-12T22:03:51.121370988Z caller=checkpoint.go:670 msg="checkpoint done" time=6m11.693324247s
level=info ts=2026-05-12T22:03:51.121389719Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-12T22:03:51.121466223Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-12T22:03:51.121471635Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T22:03:51.121483457Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=7.653µs success=true
level=debug ts=2026-05-12T22:03:51.121714306Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=debug ts=2026-05-12T22:03:52.632191132Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:03:54.427500434Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:03:54.539845739Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=debug ts=2026-05-12T22:03:57.95728578Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-12T22:03:58.54499423Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:03:59.427422047Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:04:01.375337298Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-12T22:04:04.431188582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:04:04.793428679Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=debug ts=2026-05-12T22:04:08.211227419Z caller=checkpoint.go:372 msg="writing series" size="326 kB"
level=info ts=2026-05-12T22:04:09.42733629Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:04:09.427345482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:04:09.42736293Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:04:09.427371779Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:04:11.628465582Z caller=checkpoint.go:372 msg="writing series" size="315 kB"
level=info ts=2026-05-12T22:04:12.853875022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:04:12.869530303Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:04:14.428177565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:04:15.046833698Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=debug ts=2026-05-12T22:04:15.046877112Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=debug ts=2026-05-12T22:04:18.46405173Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=warn ts=2026-05-12T22:04:19.428032532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:04:20.001050401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:04:21.881814005Z caller=checkpoint.go:372 msg="writing series" size="218 kB"
level=warn ts=2026-05-12T22:04:24.427353503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:04:25.300041466Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=debug ts=2026-05-12T22:04:28.630604482Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:04:28.716929081Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=warn ts=2026-05-12T22:04:29.42749323Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:04:32.134620505Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-12T22:04:34.428166873Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:04:35.552332642Z caller=checkpoint.go:372 msg="writing series" size="491 kB"
level=debug ts=2026-05-12T22:04:35.552358421Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=info ts=2026-05-12T22:04:37.041441045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="39 MB" avg_comp="2.4 MB" total_uncomp="318 MB" avg_uncomp="20 MB" full=15 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:04:37.052183366Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:04:38.970809292Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-12T22:04:39.411107865Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:04:39.411495449Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:04:39.427900212Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:04:39.427916795Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:04:39.42792451Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:04:39.427957227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:04:42.425571504Z caller=checkpoint.go:372 msg="writing series" size="41 MB"
level=debug ts=2026-05-12T22:04:42.425601073Z caller=checkpoint.go:384 msg="flushing series" totalSize="41 MB" series=2
level=warn ts=2026-05-12T22:04:44.428156038Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:04:45.805976807Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=debug ts=2026-05-12T22:04:49.22356558Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-05-12T22:04:49.427964151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:04:50.003885813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:04:52.023786786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:44836"
level=debug ts=2026-05-12T22:04:52.641926249Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T22:04:52.641946802Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=debug ts=2026-05-12T22:04:53.395981936Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:04:54.427869075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:04:56.061501558Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-12T22:04:56.061524964Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=1
level=warn ts=2026-05-12T22:04:59.428258784Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:04:59.476418752Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=debug ts=2026-05-12T22:05:00.284183885Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:05:02.894783347Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-12T22:05:04.427683615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:05:06.317948583Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-12T22:05:06.317974899Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=3
level=info ts=2026-05-12T22:05:09.428002366Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:05:09.428033095Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:05:09.428042606Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:05:09.428045362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:05:09.730181001Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=info ts=2026-05-12T22:05:12.460312119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:05:12.471262143Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:05:13.147160939Z caller=checkpoint.go:372 msg="writing series" size="165 kB"
level=warn ts=2026-05-12T22:05:14.427267941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:05:16.566583848Z caller=checkpoint.go:372 msg="writing series" size="2.9 MB"
level=debug ts=2026-05-12T22:05:16.566607494Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=3
level=debug ts=2026-05-12T22:05:18.550381773Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:05:19.427552946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:05:19.983343012Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
ts=2026-05-12T22:05:20.006568528Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:05:22.024976517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:51320"
level=debug ts=2026-05-12T22:05:23.401195742Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-12T22:05:24.427712645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:05:26.818534765Z caller=checkpoint.go:372 msg="writing series" size="329 kB"
level=warn ts=2026-05-12T22:05:29.427331107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:05:30.235602857Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=info ts=2026-05-12T22:05:31.102199077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="39 MB" avg_comp="2.4 MB" total_uncomp="318 MB" avg_uncomp="20 MB" full=15 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:05:31.110919253Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:05:33.654279986Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=warn ts=2026-05-12T22:05:34.42765388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:05:37.071921319Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-12T22:05:39.41097196Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:05:39.410976308Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:05:39.42744116Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:05:39.427448373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:05:39.427468053Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:05:39.427479986Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:05:40.489793621Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-12T22:05:43.907474025Z caller=checkpoint.go:372 msg="writing series" size="528 kB"
level=debug ts=2026-05-12T22:05:43.907496669Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-12T22:05:44.427362526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:05:46.715823924Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:05:46.717546832Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-12T22:05:47.325125264Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-12T22:05:49.427834606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:05:50.00877219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:05:50.708199609Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:05:50.742360562Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
ts=2026-05-12T22:05:52.026286715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:45042"
level=debug ts=2026-05-12T22:05:54.160600082Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=warn ts=2026-05-12T22:05:54.427716322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:05:57.57789789Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-12T22:05:59.172128606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:05:59.182591272Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:05:59.427203272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:06:00.996607901Z caller=checkpoint.go:372 msg="writing series" size="531 kB"
level=debug ts=2026-05-12T22:06:04.413265907Z caller=checkpoint.go:372 msg="writing series" size="295 kB"
level=debug ts=2026-05-12T22:06:04.413289807Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-05-12T22:06:04.428169259Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:06:07.831761713Z caller=checkpoint.go:372 msg="writing series" size="614 kB"
level=info ts=2026-05-12T22:06:09.428112058Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:06:09.428137209Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:06:09.428145077Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:06:09.428145078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:06:11.24851347Z caller=checkpoint.go:372 msg="writing series" size="514 B"
level=info ts=2026-05-12T22:06:11.356810177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="42 MB" avg_comp="2.5 MB" total_uncomp="341 MB" avg_uncomp="20 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:06:11.366757647Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:06:11.36678171Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:06:11.36681711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="42 MB" avg_comp="2.5 MB" total_uncomp="341 MB" avg_uncomp="20 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:06:11.375260972Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:06:14.428275649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:06:14.666326897Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-12T22:06:18.084267078Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=warn ts=2026-05-12T22:06:19.428128332Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:06:20.010543265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T22:06:21.061730778Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:38772"
level=debug ts=2026-05-12T22:06:21.502805634Z caller=checkpoint.go:372 msg="writing series" size="637 kB"
level=debug ts=2026-05-12T22:06:21.502827581Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=warn ts=2026-05-12T22:06:24.428284507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:06:24.919896369Z caller=checkpoint.go:372 msg="writing series" size="978 kB"
level=debug ts=2026-05-12T22:06:25.201931066Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:06:28.337020081Z caller=checkpoint.go:372 msg="writing series" size="141 kB"
level=debug ts=2026-05-12T22:06:28.337043612Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=warn ts=2026-05-12T22:06:29.428194791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:06:30.827119655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="42 MB" avg_comp="2.5 MB" total_uncomp="341 MB" avg_uncomp="20 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:06:30.838906853Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:06:31.755328005Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-05-12T22:06:34.427958402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:06:35.172900196Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=debug ts=2026-05-12T22:06:35.929854696Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:06:38.590187248Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-12T22:06:39.410814883Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:06:39.412012614Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=warn ts=2026-05-12T22:06:39.427339195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:06:39.427360815Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:06:39.427386597Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:06:39.42739742Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:06:42.008449663Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=warn ts=2026-05-12T22:06:44.4281543Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:06:45.425963735Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=debug ts=2026-05-12T22:06:48.843623632Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
level=debug ts=2026-05-12T22:06:49.350915419Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:06:49.350976517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:06:49.357597942Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:06:49.427664539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:06:50.013182571Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=info ts=2026-05-12T22:06:50.892953729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="42 MB" avg_comp="2.5 MB" total_uncomp="341 MB" avg_uncomp="20 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:06:50.900548894Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:06:52.261903616Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
level=warn ts=2026-05-12T22:06:54.42825171Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:06:54.784262389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:06:54.791568253Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:06:55.67893048Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=debug ts=2026-05-12T22:06:59.096778833Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
level=warn ts=2026-05-12T22:06:59.427593723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:07:01.885924045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:07:01.894187286Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:07:02.516586377Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-12T22:07:02.51660884Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=10
level=warn ts=2026-05-12T22:07:04.428091249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:07:05.933768576Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-12T22:07:09.349999902Z caller=checkpoint.go:372 msg="writing series" size="178 kB"
level=debug ts=2026-05-12T22:07:09.350022438Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-05-12T22:07:09.428013949Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:07:09.428024594Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:07:09.428039805Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:07:09.428050425Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:07:12.768089282Z caller=checkpoint.go:372 msg="writing series" size="792 kB"
level=warn ts=2026-05-12T22:07:14.428013123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:07:14.752434027Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:07:16.186176888Z caller=checkpoint.go:372 msg="writing series" size="459 kB"
level=debug ts=2026-05-12T22:07:16.186198206Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=warn ts=2026-05-12T22:07:19.427904216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:07:19.603509607Z caller=checkpoint.go:372 msg="writing series" size="159 kB"
ts=2026-05-12T22:07:20.016449022Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:07:23.021389651Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=warn ts=2026-05-12T22:07:24.428218226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:07:26.439059246Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-12T22:07:27.122940148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:07:27.131700395Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-12T22:07:27.986725299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:57606"
level=warn ts=2026-05-12T22:07:29.428071089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:07:29.856145635Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
level=debug ts=2026-05-12T22:07:33.273944499Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=warn ts=2026-05-12T22:07:34.427733346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:07:35.955385086Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:07:36.691639618Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=info ts=2026-05-12T22:07:39.410845208Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:07:39.410899673Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=warn ts=2026-05-12T22:07:39.427294749Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:07:39.427328323Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:07:39.427348872Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:07:39.427357723Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:07:40.110021032Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-12T22:07:40.110045894Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=7
level=info ts=2026-05-12T22:07:41.201675762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="42 MB" avg_comp="2.5 MB" total_uncomp="341 MB" avg_uncomp="20 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:07:41.210316105Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:07:43.527905537Z caller=checkpoint.go:372 msg="writing series" size="501 kB"
level=info ts=2026-05-12T22:07:43.746344204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:07:43.752999939Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:07:44.427877238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:07:46.283421905Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:07:46.944697017Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=warn ts=2026-05-12T22:07:49.427342764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:07:50.019842862Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:07:50.36258552Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
ts=2026-05-12T22:07:51.067580393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:57334"
ts=2026-05-12T22:07:52.038003675Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:46710"
level=debug ts=2026-05-12T22:07:53.780013347Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=debug ts=2026-05-12T22:07:53.780038186Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-05-12T22:07:54.427690918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:07:57.198589807Z caller=checkpoint.go:372 msg="writing series" size="178 kB"
level=warn ts=2026-05-12T22:07:59.428084273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:08:00.61679428Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-12T22:08:00.616818636Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=2
level=debug ts=2026-05-12T22:08:04.033426859Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=warn ts=2026-05-12T22:08:04.427775729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:08:07.451583649Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-12T22:08:09.427352043Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:08:09.427376381Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:08:09.427398023Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:08:09.427406521Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:08:10.450578704Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:08:10.869293992Z caller=checkpoint.go:372 msg="writing series" size="282 kB"
level=debug ts=2026-05-12T22:08:13.803100419Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:08:14.287171639Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-12T22:08:14.427658841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:08:15.918787703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:08:15.925197701Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:08:17.704147387Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-12T22:08:19.427727329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:08:20.022795003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:08:21.121924674Z caller=checkpoint.go:384 msg="flushing series" totalSize="355 kB" series=5
level=info ts=2026-05-12T22:08:21.128656162Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-12T22:08:21.134525006Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.012918098s
level=info ts=2026-05-12T22:08:21.134568538Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-12T22:08:21.134687173Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-12T22:08:21.134697723Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T22:08:21.134716873Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.517µs success=true
level=debug ts=2026-05-12T22:08:21.135057141Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=warn ts=2026-05-12T22:08:24.427245867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:08:24.55273764Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-05-12T22:08:26.525940952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="42 MB" avg_comp="2.5 MB" total_uncomp="341 MB" avg_uncomp="20 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:08:26.538160121Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:08:27.970799171Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-12T22:08:29.061096241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:08:29.068031138Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:08:29.06805233Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:08:29.068077473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:08:29.074812521Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:08:29.428120002Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:08:31.388322414Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-12T22:08:34.427480192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:08:34.806058462Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-12T22:08:38.224341559Z caller=checkpoint.go:372 msg="writing series" size="428 kB"
level=debug ts=2026-05-12T22:08:39.363287676Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:08:39.410821037Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:08:39.411840404Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:08:39.42719773Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:08:39.427214269Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:08:39.427223653Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:08:39.427214391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:08:41.642247952Z caller=checkpoint.go:372 msg="writing series" size="436 kB"
level=debug ts=2026-05-12T22:08:41.642271932Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-12T22:08:42.771317818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:08:42.781874684Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:08:44.428323497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:08:45.060120282Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=debug ts=2026-05-12T22:08:48.477224172Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=warn ts=2026-05-12T22:08:49.427199008Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:08:50.026189913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:08:51.073790969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:34538"
level=debug ts=2026-05-12T22:08:51.895539163Z caller=checkpoint.go:372 msg="writing series" size="269 kB"
level=warn ts=2026-05-12T22:08:54.427340622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:08:55.312906689Z caller=checkpoint.go:372 msg="writing series" size="634 kB"
level=debug ts=2026-05-12T22:08:55.312927914Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
ts=2026-05-12T22:08:57.994398696Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:57528"
level=debug ts=2026-05-12T22:08:58.731078395Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=warn ts=2026-05-12T22:08:59.42734743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:09:01.723282187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:09:01.732098441Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:09:02.148105811Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-12T22:09:04.427706888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:09:05.565715474Z caller=checkpoint.go:372 msg="writing series" size="384 kB"
level=debug ts=2026-05-12T22:09:07.426026648Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:09:08.255488179Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:09:08.983942404Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-12T22:09:09.428036176Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:09:09.428068149Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:09:09.428079891Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:09:09.42808501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:09:12.444281084Z caller=checkpoint.go:372 msg="writing series" size="45 MB"
level=debug ts=2026-05-12T22:09:12.444326824Z caller=checkpoint.go:384 msg="flushing series" totalSize="45 MB" series=5
level=info ts=2026-05-12T22:09:12.833705676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:09:12.846035725Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:09:14.427900236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:09:15.818807996Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=debug ts=2026-05-12T22:09:19.237409546Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-05-12T22:09:19.427574875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:09:20.029808253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=debug ts=2026-05-12T22:09:22.655895275Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-12T22:09:22.655922468Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=3
level=info ts=2026-05-12T22:09:23.339990005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="42 MB" avg_comp="2.5 MB" total_uncomp="341 MB" avg_uncomp="20 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:09:23.352094335Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:09:24.428186895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:09:26.074280156Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-12T22:09:26.074347131Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=1
level=warn ts=2026-05-12T22:09:29.427220195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:09:29.490485463Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=debug ts=2026-05-12T22:09:32.907399806Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-12T22:09:34.427572273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:09:35.040869844Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:09:36.337223845Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-12T22:09:36.337246345Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=3
level=info ts=2026-05-12T22:09:39.411135309Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:09:39.412089063Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:09:39.427467471Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:09:39.427488991Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:09:39.427496762Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:09:39.427525092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:09:39.743001111Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=debug ts=2026-05-12T22:09:43.160855479Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=warn ts=2026-05-12T22:09:44.428095318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:09:46.580703148Z caller=checkpoint.go:372 msg="writing series" size="2.9 MB"
level=debug ts=2026-05-12T22:09:46.580723428Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=3
level=warn ts=2026-05-12T22:09:49.427959419Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:09:49.996875161Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
ts=2026-05-12T22:09:50.032497641Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T22:09:51.078919492Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:55780"
level=info ts=2026-05-12T22:09:53.347726813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:09:53.3569093Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:09:53.413495676Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-12T22:09:54.427933821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:09:55.526890024Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:09:55.528505535Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-12T22:09:56.008843323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:09:56.016648976Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:09:56.832199877Z caller=checkpoint.go:372 msg="writing series" size="171 kB"
level=debug ts=2026-05-12T22:09:59.267542771Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:09:59.427929047Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:10:00.249388605Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=debug ts=2026-05-12T22:10:03.668469367Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=warn ts=2026-05-12T22:10:04.427863997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:10:05.799093538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="45 MB" avg_comp="2.5 MB" total_uncomp="366 MB" avg_uncomp="20 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:10:05.806793203Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:10:07.085758652Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-12T22:10:09.427744251Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:10:09.427774864Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:10:09.42778548Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:10:09.427804262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:10:10.502380147Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-12T22:10:13.920898499Z caller=checkpoint.go:372 msg="writing series" size="528 kB"
level=warn ts=2026-05-12T22:10:14.427251251Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:10:17.338363225Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-12T22:10:17.338388898Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-05-12T22:10:19.427853527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:10:20.035341091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=debug ts=2026-05-12T22:10:20.755866222Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-05-12T22:10:24.173832929Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=warn ts=2026-05-12T22:10:24.427436582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:10:27.59162785Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-05-12T22:10:29.427207415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:10:31.009431747Z caller=checkpoint.go:372 msg="writing series" size="420 kB"
level=debug ts=2026-05-12T22:10:32.936618958Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:10:34.426896388Z caller=checkpoint.go:372 msg="writing series" size="899 kB"
level=debug ts=2026-05-12T22:10:34.42691704Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=warn ts=2026-05-12T22:10:34.427343506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:10:37.459819033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:10:37.466874629Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:10:37.844549772Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=debug ts=2026-05-12T22:10:39.302051062Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:10:39.410520799Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:10:39.410940226Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:10:39.427390015Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:10:39.427412161Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:10:39.427421732Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:10:39.427425371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:10:41.261924276Z caller=checkpoint.go:372 msg="writing series" size="642 B"
level=info ts=2026-05-12T22:10:43.050836869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:10:43.057483947Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:10:44.427777616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:10:44.679818481Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=debug ts=2026-05-12T22:10:48.097414345Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=warn ts=2026-05-12T22:10:49.428223637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:10:50.038801752Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:10:51.515860524Z caller=checkpoint.go:372 msg="writing series" size="709 kB"
level=debug ts=2026-05-12T22:10:51.515883719Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=info ts=2026-05-12T22:10:54.401865296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="45 MB" avg_comp="2.5 MB" total_uncomp="366 MB" avg_uncomp="20 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:10:54.41324564Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:10:54.427287611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:10:54.933409913Z caller=checkpoint.go:372 msg="writing series" size="747 kB"
level=debug ts=2026-05-12T22:10:57.314513695Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-12T22:10:58.002384533Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:54418"
level=debug ts=2026-05-12T22:10:58.350708381Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=warn ts=2026-05-12T22:10:59.427939602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:11:01.768040745Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-05-12T22:11:04.4278116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:11:05.186228683Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=debug ts=2026-05-12T22:11:08.604354536Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-12T22:11:09.42735125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:11:09.427368811Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:11:09.42739007Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:11:09.427398363Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:11:12.02167143Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=debug ts=2026-05-12T22:11:12.021694486Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=warn ts=2026-05-12T22:11:14.427714891Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:11:15.439531064Z caller=checkpoint.go:372 msg="writing series" size="6.6 kB"
level=debug ts=2026-05-12T22:11:18.857122089Z caller=checkpoint.go:372 msg="writing series" size="309 kB"
level=warn ts=2026-05-12T22:11:19.427404189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:11:20.04233144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:11:21.085687036Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:45740"
level=debug ts=2026-05-12T22:11:22.274717034Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=info ts=2026-05-12T22:11:22.490424616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="97 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:11:22.501681757Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:11:24.190705985Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:11:24.427261378Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:11:25.692361734Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=debug ts=2026-05-12T22:11:29.109740235Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=warn ts=2026-05-12T22:11:29.427557479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:11:32.530279166Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-12T22:11:32.530370375Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=6
level=warn ts=2026-05-12T22:11:34.427422506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:11:35.557716577Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:11:35.946774962Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-12T22:11:35.946796653Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=1
level=info ts=2026-05-12T22:11:39.093737721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:11:39.10124497Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:11:39.363271603Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=debug ts=2026-05-12T22:11:39.410721209Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:11:39.410974139Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-12T22:11:39.427354217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:11:39.427356162Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:11:39.42737714Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:11:39.427384733Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T22:11:42.118252391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="45 MB" avg_comp="2.5 MB" total_uncomp="366 MB" avg_uncomp="20 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:11:42.128790268Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:11:42.781407709Z caller=checkpoint.go:372 msg="writing series" size="792 kB"
level=warn ts=2026-05-12T22:11:44.42728499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:11:46.199334345Z caller=checkpoint.go:372 msg="writing series" size="537 kB"
level=debug ts=2026-05-12T22:11:46.19935886Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=warn ts=2026-05-12T22:11:49.427213821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:11:49.616752846Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
ts=2026-05-12T22:11:50.045986262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=debug ts=2026-05-12T22:11:52.317075474Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:11:53.034210129Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
level=warn ts=2026-05-12T22:11:54.42777155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:11:56.452059654Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-12T22:11:59.427775978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:11:59.870056738Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=debug ts=2026-05-12T22:12:03.287380993Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=warn ts=2026-05-12T22:12:04.427641675Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:12:04.878352664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="97 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:12:04.887902786Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:12:06.705353756Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=info ts=2026-05-12T22:12:09.427320869Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:12:09.427346961Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:12:09.427355142Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:12:09.42737426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:12:10.123183111Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-12T22:12:10.123203335Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=7
level=debug ts=2026-05-12T22:12:13.540475731Z caller=checkpoint.go:372 msg="writing series" size="647 kB"
level=warn ts=2026-05-12T22:12:14.427353972Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:12:16.651393556Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:12:16.958464093Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=debug ts=2026-05-12T22:12:16.958484958Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=warn ts=2026-05-12T22:12:19.427205372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:12:19.538578588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:12:19.545789845Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-12T22:12:20.049466709Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=debug ts=2026-05-12T22:12:20.376313328Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-12T22:12:22.453062576Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:12:23.793552765Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=warn ts=2026-05-12T22:12:24.427712494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:12:25.450182764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="45 MB" avg_comp="2.5 MB" total_uncomp="366 MB" avg_uncomp="20 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:12:25.457803775Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:12:25.457822972Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:12:25.457856518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="45 MB" avg_comp="2.5 MB" total_uncomp="366 MB" avg_uncomp="20 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:12:25.463977544Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:12:27.211909886Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
ts=2026-05-12T22:12:28.007380978Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:44252"
level=warn ts=2026-05-12T22:12:29.427204872Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:12:30.631628134Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-12T22:12:30.631657901Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=4
level=debug ts=2026-05-12T22:12:34.046673516Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=warn ts=2026-05-12T22:12:34.427529677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:12:37.465117033Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-05-12T22:12:37.918507373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="45 MB" avg_comp="2.5 MB" total_uncomp="366 MB" avg_uncomp="20 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:12:37.929919917Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:12:39.411145129Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:12:39.411583289Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:12:39.42789971Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:12:39.427901326Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:12:39.427917912Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:12:39.427925381Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:12:40.881986862Z caller=checkpoint.go:372 msg="writing series" size="282 kB"
level=debug ts=2026-05-12T22:12:44.299649627Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-12T22:12:44.428146912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:12:46.028863365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="97 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:12:46.037666313Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:12:47.717649875Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-12T22:12:49.428087734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:12:50.052193885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=debug ts=2026-05-12T22:12:50.233428153Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-12T22:12:51.091765508Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:36590"
level=debug ts=2026-05-12T22:12:51.135498057Z caller=checkpoint.go:384 msg="flushing series" totalSize="356 kB" series=5
level=info ts=2026-05-12T22:12:51.140972577Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-12T22:12:51.143537437Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.00861522s
level=info ts=2026-05-12T22:12:51.143560854Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-12T22:12:51.143639833Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-12T22:12:51.143646269Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T22:12:51.143658034Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=7.74µs success=true
level=debug ts=2026-05-12T22:12:51.143883515Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=warn ts=2026-05-12T22:12:54.428330702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:12:54.562629875Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=debug ts=2026-05-12T22:12:57.979326686Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-12T22:12:59.427961009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:13:01.397431413Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-12T22:13:04.42734362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:13:04.814976954Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-12T22:13:08.232832954Z caller=checkpoint.go:372 msg="writing series" size="337 kB"
level=info ts=2026-05-12T22:13:09.427965629Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:13:09.42799353Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:13:09.428001033Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:13:09.428039533Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:13:11.65098172Z caller=checkpoint.go:372 msg="writing series" size="357 kB"
level=warn ts=2026-05-12T22:13:14.428040827Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:13:15.068252583Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=debug ts=2026-05-12T22:13:15.068281825Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-05-12T22:13:15.738524979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="45 MB" avg_comp="2.5 MB" total_uncomp="366 MB" avg_uncomp="20 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:13:15.748829987Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:13:15.778959762Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:13:17.58724266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:13:17.593558741Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:13:17.593583201Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:13:17.593606304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:13:17.599741235Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:13:18.113650805Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:13:18.485686905Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=warn ts=2026-05-12T22:13:19.427387172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:13:20.055687449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T22:13:21.09453301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:60482"
level=debug ts=2026-05-12T22:13:21.903401224Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=warn ts=2026-05-12T22:13:24.428100495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:13:25.321697379Z caller=checkpoint.go:372 msg="writing series" size="740 kB"
level=debug ts=2026-05-12T22:13:28.7397133Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=warn ts=2026-05-12T22:13:29.427781052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:13:29.706025256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:13:29.715486751Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:13:32.160494814Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-12T22:13:34.427532847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:13:35.57485732Z caller=checkpoint.go:372 msg="writing series" size="466 kB"
level=debug ts=2026-05-12T22:13:35.574880611Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=debug ts=2026-05-12T22:13:38.992825012Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-12T22:13:39.410866597Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:13:39.41128995Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:13:39.427615909Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:13:39.427633386Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-12T22:13:39.427630883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:13:39.427644407Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T22:13:40.183560181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="97 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:13:40.195567546Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:13:42.454734789Z caller=checkpoint.go:372 msg="writing series" size="46 MB"
level=debug ts=2026-05-12T22:13:42.45476365Z caller=checkpoint.go:384 msg="flushing series" totalSize="46 MB" series=2
level=warn ts=2026-05-12T22:13:44.428056384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:13:45.827883397Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=debug ts=2026-05-12T22:13:49.246018514Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-05-12T22:13:49.428259451Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:13:50.057484886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:13:50.598446925Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-12T22:13:51.0982144Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:58116"
level=debug ts=2026-05-12T22:13:52.664668837Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-12T22:13:52.664689299Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=3
level=warn ts=2026-05-12T22:13:54.428212441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:13:56.082786665Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-12T22:13:56.082806716Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=1
level=info ts=2026-05-12T22:13:57.900806652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:13:57.909471499Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-12T22:13:58.014891543Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:47424"
level=warn ts=2026-05-12T22:13:59.427361149Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:13:59.498626334Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-05-12T22:14:02.916624025Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-12T22:14:04.427233813Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:14:06.342906338Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-12T22:14:06.342926652Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=3
level=warn ts=2026-05-12T22:14:09.427483979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:14:09.427505341Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:14:09.427526416Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:14:09.427534912Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:14:09.752106399Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=debug ts=2026-05-12T22:14:09.839597893Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:14:09.841344574Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-12T22:14:11.804456579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="45 MB" avg_comp="2.5 MB" total_uncomp="366 MB" avg_uncomp="20 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:14:11.815292445Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:14:13.169678268Z caller=checkpoint.go:372 msg="writing series" size="262 kB"
level=warn ts=2026-05-12T22:14:14.427564787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:14:16.589895132Z caller=checkpoint.go:372 msg="writing series" size="2.9 MB"
level=debug ts=2026-05-12T22:14:16.589921058Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=3
level=warn ts=2026-05-12T22:14:19.428038338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:14:20.005515197Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
ts=2026-05-12T22:14:20.059890404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T22:14:22.078260827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:52580"
level=debug ts=2026-05-12T22:14:22.344389048Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:14:23.422623987Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-12T22:14:24.427798988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:14:26.840854658Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
level=warn ts=2026-05-12T22:14:29.427320424Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:14:30.258679822Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=debug ts=2026-05-12T22:14:33.676053259Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=warn ts=2026-05-12T22:14:34.428250778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:14:37.093432693Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-12T22:14:37.434759732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="97 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:14:37.443684312Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:14:37.443708582Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:14:37.443734621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="97 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:14:37.450768174Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:14:39.410852489Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:14:39.411488811Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:14:39.427758442Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:14:39.427776556Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:14:39.427784304Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:14:39.427801171Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:14:40.511729354Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-12T22:14:40.547204676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:14:40.554453809Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:14:43.930138496Z caller=checkpoint.go:372 msg="writing series" size="528 kB"
level=debug ts=2026-05-12T22:14:43.930160334Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-12T22:14:44.428292372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:14:47.346946683Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-12T22:14:49.427668039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:14:50.062933172Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=debug ts=2026-05-12T22:14:50.764995216Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
ts=2026-05-12T22:14:52.079198141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:49404"
level=debug ts=2026-05-12T22:14:54.182860046Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=warn ts=2026-05-12T22:14:54.428212716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:14:54.435384884Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:14:55.282564436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="97 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:14:55.294712171Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:14:57.600027882Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-05-12T22:14:59.42814307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:15:01.017985427Z caller=checkpoint.go:372 msg="writing series" size="420 kB"
level=info ts=2026-05-12T22:15:02.11152719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="45 MB" avg_comp="2.5 MB" total_uncomp="366 MB" avg_uncomp="20 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:15:02.11957399Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:15:04.427848507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:15:04.436569789Z caller=checkpoint.go:372 msg="writing series" size="899 kB"
level=debug ts=2026-05-12T22:15:04.436585707Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=6
level=debug ts=2026-05-12T22:15:07.853206587Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=info ts=2026-05-12T22:15:09.427975695Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:15:09.428008523Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-12T22:15:09.428006163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:15:09.428019161Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:15:11.27145106Z caller=checkpoint.go:372 msg="writing series" size="642 B"
level=debug ts=2026-05-12T22:15:11.490621696Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:15:14.427747742Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:15:14.688651735Z caller=checkpoint.go:372 msg="writing series" size="3.6 kB"
level=info ts=2026-05-12T22:15:17.380601414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="97 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:15:17.389238963Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:15:18.106552071Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=warn ts=2026-05-12T22:15:19.428146767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:15:20.065468079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:15:21.108065297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:41466"
level=debug ts=2026-05-12T22:15:21.524180635Z caller=checkpoint.go:372 msg="writing series" size="740 kB"
level=debug ts=2026-05-12T22:15:21.524202701Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=warn ts=2026-05-12T22:15:24.427908532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:15:24.942054709Z caller=checkpoint.go:372 msg="writing series" size="802 kB"
level=debug ts=2026-05-12T22:15:28.359184411Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=debug ts=2026-05-12T22:15:28.360192227Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:15:29.427939039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:15:31.777526287Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-05-12T22:15:34.428140581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:15:35.194835238Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-12T22:15:36.422833629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:15:36.431366147Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:15:38.612751899Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-12T22:15:38.612774998Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-12T22:15:39.410877212Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:15:39.411334555Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=debug ts=2026-05-12T22:15:39.412585072Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-12T22:15:39.41263196Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976248 n=1
level=debug ts=2026-05-12T22:15:39.412680242Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976248 n=1
level=info ts=2026-05-12T22:15:39.427664678Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:15:39.427681398Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-12T22:15:39.427686403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:15:39.427693372Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:15:42.030533867Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=warn ts=2026-05-12T22:15:44.428185122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:15:45.447966681Z caller=checkpoint.go:372 msg="writing series" size="6.6 kB"
level=debug ts=2026-05-12T22:15:48.866132323Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=warn ts=2026-05-12T22:15:49.427318186Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:15:50.068870211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:15:51.4226098Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:15:52.283732417Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=warn ts=2026-05-12T22:15:54.428185191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:15:54.567528031Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:15:55.701429377Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
ts=2026-05-12T22:15:58.026077266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:47604"
level=info ts=2026-05-12T22:15:58.53905519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="45 MB" avg_comp="2.5 MB" total_uncomp="366 MB" avg_uncomp="20 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:15:58.547012038Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:15:59.118664942Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=warn ts=2026-05-12T22:15:59.427815707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:16:02.539141095Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-12T22:16:02.539163524Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=7
level=warn ts=2026-05-12T22:16:04.427722159Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:16:05.955179187Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T22:16:05.955201014Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
level=debug ts=2026-05-12T22:16:09.372340115Z caller=checkpoint.go:372 msg="writing series" size="213 kB"
level=info ts=2026-05-12T22:16:09.427165013Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:16:09.427187048Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:16:09.42719467Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:16:09.427224186Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:16:11.78691773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="97 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:16:11.796745008Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:16:12.790667926Z caller=checkpoint.go:372 msg="writing series" size="792 kB"
level=warn ts=2026-05-12T22:16:14.427602447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:16:16.207364707Z caller=checkpoint.go:372 msg="writing series" size="537 kB"
level=debug ts=2026-05-12T22:16:16.20738842Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=warn ts=2026-05-12T22:16:19.427824249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:16:19.626005882Z caller=checkpoint.go:372 msg="writing series" size="217 kB"
ts=2026-05-12T22:16:20.071365427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:16:20.626867055Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:16:23.042848913Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=warn ts=2026-05-12T22:16:24.427515094Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:16:26.460624296Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-05-12T22:16:27.01010797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:16:27.020807795Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:16:29.427252645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:16:29.878145133Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=debug ts=2026-05-12T22:16:33.180854383Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:16:33.296269349Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=warn ts=2026-05-12T22:16:34.427478612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:16:36.713803349Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=info ts=2026-05-12T22:16:38.85169074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="45 MB" avg_comp="2.5 MB" total_uncomp="366 MB" avg_uncomp="20 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:16:38.861179375Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:16:39.410829058Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:16:39.411182153Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:16:39.427474823Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:16:39.427494918Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:16:39.427503665Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:16:39.427509056Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:16:40.133450435Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-12T22:16:40.133470441Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=7
level=debug ts=2026-05-12T22:16:43.550162232Z caller=checkpoint.go:372 msg="writing series" size="672 kB"
level=warn ts=2026-05-12T22:16:44.427552332Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:16:46.967694958Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=debug ts=2026-05-12T22:16:46.967719109Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=debug ts=2026-05-12T22:16:47.228399943Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:16:49.427225484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:16:50.073603347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:16:50.3845019Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-12T22:16:53.802924865Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=warn ts=2026-05-12T22:16:54.428053874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:16:57.220636148Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=warn ts=2026-05-12T22:16:59.42743912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:17:00.64071424Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-12T22:17:00.640736415Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=4
level=debug ts=2026-05-12T22:17:04.055783742Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=warn ts=2026-05-12T22:17:04.427725077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:17:05.08979757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="97 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:17:05.099481335Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:17:07.473668421Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-12T22:17:09.427250504Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:17:09.427275871Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:17:09.427283394Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:17:09.427315337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:17:10.891719337Z caller=checkpoint.go:372 msg="writing series" size="282 kB"
level=debug ts=2026-05-12T22:17:13.395374005Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:17:14.309182019Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-12T22:17:14.427936773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:17:17.726281959Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-12T22:17:19.428015869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:17:20.076249496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=info ts=2026-05-12T22:17:20.516571205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:17:20.524113078Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:17:21.144749974Z caller=checkpoint.go:384 msg="flushing series" totalSize="357 kB" series=5
level=info ts=2026-05-12T22:17:21.147576394Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-12T22:17:21.149508867Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.005728816s
level=info ts=2026-05-12T22:17:23.894093506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="45 MB" avg_comp="2.5 MB" total_uncomp="366 MB" avg_uncomp="20 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:17:23.901424279Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:17:24.427996594Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:17:27.201900834Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:17:29.427764629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T22:17:34.427770878Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:17:39.411221371Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:17:39.412074733Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:17:39.427258503Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-12T22:17:39.427320592Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:17:39.427340422Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:17:39.427350909Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:17:39.427346501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:17:39.427491353Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-12T22:17:39.427500524Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T22:17:39.427519006Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.92µs success=true
level=debug ts=2026-05-12T22:17:39.427739204Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=debug ts=2026-05-12T22:17:41.03665797Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:17:42.845686103Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-12T22:17:44.427620675Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:17:46.263367791Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-12T22:17:47.32778241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="97 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:17:47.334971883Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:17:49.427815257Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:17:49.68174837Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
ts=2026-05-12T22:17:50.077580791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T22:17:52.09511719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:39296"
level=debug ts=2026-05-12T22:17:53.099192985Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-12T22:17:54.427820176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:17:56.516693786Z caller=checkpoint.go:372 msg="writing series" size="444 kB"
level=warn ts=2026-05-12T22:17:59.42779463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:17:59.93482614Z caller=checkpoint.go:372 msg="writing series" size="481 kB"
level=debug ts=2026-05-12T22:17:59.934846501Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=debug ts=2026-05-12T22:18:03.352446714Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=warn ts=2026-05-12T22:18:04.427819764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:18:06.77034452Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=info ts=2026-05-12T22:18:09.427232862Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:18:09.427259143Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:18:09.427266928Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:18:09.427276498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:18:09.816567901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="45 MB" avg_comp="2.5 MB" total_uncomp="366 MB" avg_uncomp="20 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:18:09.824447731Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:18:10.1876874Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=debug ts=2026-05-12T22:18:11.265351873Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:18:13.605251195Z caller=checkpoint.go:372 msg="writing series" size="702 kB"
level=debug ts=2026-05-12T22:18:13.605272681Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-05-12T22:18:14.42784186Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:18:15.059232674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:18:15.065327478Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:18:15.976583663Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:18:15.978136391Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-12T22:18:17.02355429Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=warn ts=2026-05-12T22:18:19.428324864Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:18:20.079235017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:18:20.441199439Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
ts=2026-05-12T22:18:21.123045175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:54594"
level=debug ts=2026-05-12T22:18:23.85933282Z caller=checkpoint.go:372 msg="writing series" size="563 kB"
level=warn ts=2026-05-12T22:18:24.428153407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:18:27.276201351Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-12T22:18:29.4279165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:18:30.737190683Z caller=checkpoint.go:372 msg="writing series" size="46 MB"
level=debug ts=2026-05-12T22:18:30.737215454Z caller=checkpoint.go:384 msg="flushing series" totalSize="47 MB" series=5
level=debug ts=2026-05-12T22:18:34.111454874Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-12T22:18:34.428314479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:18:37.530096462Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-05-12T22:18:39.411500359Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:18:39.412292932Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:18:39.42745509Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:18:39.427472371Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:18:39.427482084Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:18:39.42751654Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:18:40.949224861Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-12T22:18:40.949247696Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=3
level=debug ts=2026-05-12T22:18:42.974213196Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:18:44.366653439Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-12T22:18:44.366679624Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=1
level=warn ts=2026-05-12T22:18:44.427699145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:18:46.402548359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="97 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:18:46.411973884Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:18:47.782414955Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=warn ts=2026-05-12T22:18:49.42789578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:18:50.081862195Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T22:18:51.126574711Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:38616"
level=debug ts=2026-05-12T22:18:51.200163193Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
ts=2026-05-12T22:18:52.099045474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:56828"
level=info ts=2026-05-12T22:18:53.290909857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="45 MB" avg_comp="2.5 MB" total_uncomp="366 MB" avg_uncomp="20 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:18:53.298946406Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:18:53.298971279Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:18:53.299006395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="45 MB" avg_comp="2.5 MB" total_uncomp="366 MB" avg_uncomp="20 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:18:53.306091347Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:18:54.427345978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:18:54.62997056Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-12T22:18:54.629996169Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=3
level=info ts=2026-05-12T22:18:55.746752376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:18:55.75497009Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:18:58.035459948Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
ts=2026-05-12T22:18:58.042890126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:53080"
level=warn ts=2026-05-12T22:18:59.427800956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:19:01.453701683Z caller=checkpoint.go:372 msg="writing series" size="312 kB"
level=warn ts=2026-05-12T22:19:04.427347859Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:19:04.873035497Z caller=checkpoint.go:372 msg="writing series" size="3.1 MB"
level=debug ts=2026-05-12T22:19:04.873055434Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.4 MB" series=3
level=debug ts=2026-05-12T22:19:08.289214036Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-12T22:19:09.355251307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="45 MB" avg_comp="2.5 MB" total_uncomp="366 MB" avg_uncomp="20 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:19:09.365611877Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-12T22:19:09.427733971Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:19:09.427753465Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:19:09.427761893Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:19:09.427751782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:19:11.706744692Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-12T22:19:14.427674238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:19:15.124398901Z caller=checkpoint.go:372 msg="writing series" size="305 kB"
level=debug ts=2026-05-12T22:19:15.407418058Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:19:15.772500768Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:19:18.542244373Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=warn ts=2026-05-12T22:19:19.427469957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:19:20.083803612Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T22:19:21.130121776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:52344"
level=debug ts=2026-05-12T22:19:21.959854358Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=warn ts=2026-05-12T22:19:24.428076474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:19:25.377253765Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
ts=2026-05-12T22:19:28.046445182Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:60206"
level=debug ts=2026-05-12T22:19:28.795831104Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-12T22:19:29.427579435Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:19:32.213872102Z caller=checkpoint.go:372 msg="writing series" size="528 kB"
level=debug ts=2026-05-12T22:19:32.213892478Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=warn ts=2026-05-12T22:19:34.427565087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:19:35.630735253Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-12T22:19:35.736007577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="97 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:19:35.748713758Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:19:36.232626679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:19:36.23936493Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:19:36.239382939Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:19:36.239406683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:19:36.2456789Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:19:39.048678931Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-12T22:19:39.410842946Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:19:39.410840774Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:19:39.427195135Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:19:39.427219273Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-12T22:19:39.427219728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:19:39.427229118Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:19:41.810019863Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:19:42.217182205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="45 MB" avg_comp="2.5 MB" total_uncomp="366 MB" avg_uncomp="20 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:19:42.225862364Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:19:42.466760151Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=warn ts=2026-05-12T22:19:44.427618765Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:19:45.884213621Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=debug ts=2026-05-12T22:19:49.301983193Z caller=checkpoint.go:372 msg="writing series" size="464 kB"
level=warn ts=2026-05-12T22:19:49.428116381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:19:50.084894125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:19:52.720201823Z caller=checkpoint.go:372 msg="writing series" size="899 kB"
level=debug ts=2026-05-12T22:19:52.720225089Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=6
level=info ts=2026-05-12T22:19:54.208584493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:19:54.218226233Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:19:54.427518725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:19:56.13783916Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=warn ts=2026-05-12T22:19:59.42822339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:19:59.554579067Z caller=checkpoint.go:372 msg="writing series" size="642 B"
level=debug ts=2026-05-12T22:20:02.972913802Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=debug ts=2026-05-12T22:20:03.744470468Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:20:04.427418567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:20:06.19371357Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:20:06.39041692Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=info ts=2026-05-12T22:20:09.427212367Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:20:09.427207936Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:20:09.427234775Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:20:09.427243031Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:20:09.808564161Z caller=checkpoint.go:372 msg="writing series" size="807 kB"
level=debug ts=2026-05-12T22:20:09.808583823Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=debug ts=2026-05-12T22:20:13.226003754Z caller=checkpoint.go:372 msg="writing series" size="885 kB"
level=warn ts=2026-05-12T22:20:14.427995289Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:20:16.643257364Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=debug ts=2026-05-12T22:20:16.64327783Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=warn ts=2026-05-12T22:20:19.427361886Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:20:19.725417031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:20:19.734099179Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:20:20.061645501Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
ts=2026-05-12T22:20:20.08593439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T22:20:22.106061961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:44160"
level=debug ts=2026-05-12T22:20:23.479370056Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-12T22:20:24.427660279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:20:26.896634647Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-12T22:20:26.946881184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="97 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:20:26.956979524Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:20:29.42722207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:20:30.314698084Z caller=checkpoint.go:372 msg="writing series" size="204 kB"
level=debug ts=2026-05-12T22:20:33.731544552Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-12T22:20:34.427502371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:20:35.000233804Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:20:37.150000558Z caller=checkpoint.go:372 msg="writing series" size="159 kB"
level=info ts=2026-05-12T22:20:39.41083819Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:20:39.412651745Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:20:39.427870615Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:20:39.42788853Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:20:39.427896532Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:20:39.427899681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:20:40.567690303Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=info ts=2026-05-12T22:20:40.716151379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="45 MB" avg_comp="2.5 MB" total_uncomp="366 MB" avg_uncomp="20 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:20:40.724542602Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:20:43.985324746Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=warn ts=2026-05-12T22:20:44.427785514Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:20:47.403503101Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=warn ts=2026-05-12T22:20:49.42782815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:20:50.087534983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:20:50.82191827Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-12T22:20:50.821941469Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=10
level=debug ts=2026-05-12T22:20:52.772906949Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:20:54.239369411Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-12T22:20:54.239390024Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=1
level=warn ts=2026-05-12T22:20:54.427607165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:20:57.656195037Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=warn ts=2026-05-12T22:20:59.427896309Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:21:01.073939921Z caller=checkpoint.go:372 msg="writing series" size="792 kB"
level=warn ts=2026-05-12T22:21:04.427557007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:21:04.492060272Z caller=checkpoint.go:372 msg="writing series" size="364 kB"
level=debug ts=2026-05-12T22:21:04.492080731Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=debug ts=2026-05-12T22:21:07.908876542Z caller=checkpoint.go:372 msg="writing series" size="247 kB"
level=debug ts=2026-05-12T22:21:08.603703315Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:21:09.427760912Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:21:09.427777991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:21:09.427793272Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:21:09.427803794Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:21:11.326688258Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=warn ts=2026-05-12T22:21:14.427349151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:21:14.725683845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="97 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:21:14.733711337Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:21:14.733734942Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:21:14.733769322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="97 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:21:14.740395564Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:21:14.744500859Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-12T22:21:17.947264225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:21:17.953811608Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:21:18.16254484Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=warn ts=2026-05-12T22:21:19.427352921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:21:20.089459199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:21:21.136831869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:57158"
level=debug ts=2026-05-12T22:21:21.580069158Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
ts=2026-05-12T22:21:22.111201182Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:45602"
level=warn ts=2026-05-12T22:21:24.428174845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:21:24.997538849Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=debug ts=2026-05-12T22:21:28.417038808Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T22:21:28.417067495Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=7
level=warn ts=2026-05-12T22:21:29.427259604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:21:29.788888264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="97 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:21:29.799114094Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:21:30.486424581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="45 MB" avg_comp="2.5 MB" total_uncomp="366 MB" avg_uncomp="20 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:21:30.495191141Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:21:31.833677629Z caller=checkpoint.go:372 msg="writing series" size="715 kB"
level=warn ts=2026-05-12T22:21:34.42802766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:21:35.250875259Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=debug ts=2026-05-12T22:21:35.250896071Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=debug ts=2026-05-12T22:21:38.668893691Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-12T22:21:39.410909119Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:21:39.410928377Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:21:39.427186593Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:21:39.427203962Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:21:39.427211226Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:21:39.427220835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:21:39.823950559Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:21:41.006290043Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:21:42.08632656Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=warn ts=2026-05-12T22:21:44.428216024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:21:45.504141162Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=debug ts=2026-05-12T22:21:48.923920055Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-12T22:21:48.923947021Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=4
level=warn ts=2026-05-12T22:21:49.428195759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:21:50.092597713Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=debug ts=2026-05-12T22:21:52.339592407Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=warn ts=2026-05-12T22:21:54.427348528Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:21:55.757564619Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-12T22:21:59.175370422Z caller=checkpoint.go:372 msg="writing series" size="282 kB"
level=warn ts=2026-05-12T22:21:59.428143547Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:21:59.9719628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:21:59.983119353Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:22:02.593249536Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-12T22:22:04.427985495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:22:06.010867296Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-12T22:22:08.334313146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="97 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:22:08.340795916Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:22:09.041930756Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:22:09.428120918Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:22:09.428122596Z caller=checkpoint.go:384 msg="flushing series" totalSize="358 kB" series=5
level=warn ts=2026-05-12T22:22:09.428135539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:22:09.428146718Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:22:09.42815505Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T22:22:09.438152612Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-12T22:22:09.440153098Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.012522124s
level=warn ts=2026-05-12T22:22:14.427353406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T22:22:19.42749955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:22:20.096252143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:22:20.572684453Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:22:20.574291246Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-12T22:22:22.222893606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="45 MB" avg_comp="2.5 MB" total_uncomp="366 MB" avg_uncomp="20 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:22:22.230922652Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:22:24.428011511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:22:28.059176771Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:36568"
level=warn ts=2026-05-12T22:22:29.427998488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-12T22:22:34.427441066Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:22:39.410886325Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:22:39.411560942Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:22:39.427851236Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-12T22:22:39.427856299Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:22:39.427873448Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:22:39.427881659Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:22:39.42788284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:22:39.427984662Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-12T22:22:39.427992573Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T22:22:39.42800834Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.202µs success=true
level=debug ts=2026-05-12T22:22:39.428218725Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=debug ts=2026-05-12T22:22:40.258403302Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:22:42.846091681Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-12T22:22:44.428192179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:22:46.263909044Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-12T22:22:46.902776509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:22:46.911429907Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:22:49.427610175Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:22:49.681626783Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
ts=2026-05-12T22:22:50.098875343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:22:52.1195244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:35758"
level=debug ts=2026-05-12T22:22:53.099548129Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-12T22:22:54.427968264Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:22:56.517927534Z caller=checkpoint.go:372 msg="writing series" size="350 kB"
level=warn ts=2026-05-12T22:22:59.427643783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:22:59.93490531Z caller=checkpoint.go:372 msg="writing series" size="402 kB"
level=info ts=2026-05-12T22:23:03.109412785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="97 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:23:03.116741911Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:23:03.352421818Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=debug ts=2026-05-12T22:23:03.352442523Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=warn ts=2026-05-12T22:23:04.428208785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:23:06.158867234Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:23:06.158915323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="501 kB" avg_comp="501 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:23:06.162951222Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:23:06.770793259Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=info ts=2026-05-12T22:23:09.427697179Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:23:09.427722733Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:23:09.427730838Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:23:09.427746752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:23:10.188031974Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=debug ts=2026-05-12T22:23:13.606250595Z caller=checkpoint.go:372 msg="writing series" size="706 kB"
level=warn ts=2026-05-12T22:23:14.427431826Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:23:16.256337841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:23:16.260711329Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:23:17.024231463Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=warn ts=2026-05-12T22:23:19.427362404Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:23:20.101721462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=info ts=2026-05-12T22:23:20.35579847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="45 MB" avg_comp="2.5 MB" total_uncomp="366 MB" avg_uncomp="20 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:23:20.3632569Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-12T22:23:20.441830788Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=debug ts=2026-05-12T22:23:20.56100363Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:23:22.120861249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:39830"
level=debug ts=2026-05-12T22:23:23.859320035Z caller=checkpoint.go:372 msg="writing series" size="496 kB"
level=debug ts=2026-05-12T22:23:23.859340361Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=warn ts=2026-05-12T22:23:24.428181984Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:23:27.276888736Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
ts=2026-05-12T22:23:28.06385079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:50752"
level=warn ts=2026-05-12T22:23:29.427231382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:23:30.741969918Z caller=checkpoint.go:372 msg="writing series" size="48 MB"
level=debug ts=2026-05-12T22:23:30.74199506Z caller=checkpoint.go:384 msg="flushing series" totalSize="48 MB" series=2
level=info ts=2026-05-12T22:23:31.628112339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:23:31.638138604Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:23:34.111993104Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-12T22:23:34.427807638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:23:37.529882481Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-05-12T22:23:38.570735731Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:23:39.410875329Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:23:39.411392284Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:23:39.427757311Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:23:39.427776096Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:23:39.427784253Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:23:39.427807403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:23:40.948818713Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-12T22:23:40.948839952Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=3
level=debug ts=2026-05-12T22:23:44.366340802Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-12T22:23:44.366362171Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=1
level=warn ts=2026-05-12T22:23:44.427885712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:23:47.782984727Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=warn ts=2026-05-12T22:23:49.427446409Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:23:49.536781496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:23:49.54302144Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-12T22:23:50.105140978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=info ts=2026-05-12T22:23:50.709537054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="97 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:23:50.718368762Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:23:51.200593675Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
ts=2026-05-12T22:23:52.122670731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:50534"
level=warn ts=2026-05-12T22:23:54.428273243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:23:54.628180828Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-05-12T22:23:54.62820443Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=3
level=debug ts=2026-05-12T22:23:58.036531007Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=warn ts=2026-05-12T22:23:59.427689563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:24:01.453786005Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=info ts=2026-05-12T22:24:02.30196092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="48 MB" avg_comp="2.5 MB" total_uncomp="390 MB" avg_uncomp="20 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:24:02.312262371Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:24:04.428111122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:24:04.873722191Z caller=checkpoint.go:372 msg="writing series" size="3.1 MB"
level=debug ts=2026-05-12T22:24:04.873743125Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=3
level=debug ts=2026-05-12T22:24:08.289413954Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-12T22:24:09.42778516Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:24:09.427814022Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:24:09.427823082Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:24:09.427871328Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:24:11.674365447Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:24:11.707606563Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-12T22:24:14.427496617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:24:15.124772779Z caller=checkpoint.go:372 msg="writing series" size="373 kB"
level=debug ts=2026-05-12T22:24:17.735840707Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:24:18.542157929Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=info ts=2026-05-12T22:24:19.240173059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:24:19.247387525Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:24:19.427617339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:24:20.108574662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:24:21.960929835Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=warn ts=2026-05-12T22:24:24.427258748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:24:25.378218694Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
ts=2026-05-12T22:24:28.068551473Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:57440"
level=debug ts=2026-05-12T22:24:28.795993394Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-12T22:24:29.428012008Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:24:32.213967413Z caller=checkpoint.go:372 msg="writing series" size="528 kB"
level=debug ts=2026-05-12T22:24:32.2139923Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=info ts=2026-05-12T22:24:33.091327817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:24:33.099329021Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T22:24:34.428244235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:24:35.630927818Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-12T22:24:39.04945127Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-12T22:24:39.410890857Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:24:39.411050379Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=warn ts=2026-05-12T22:24:39.427359089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:24:39.427360301Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:24:39.427384589Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:24:39.427392578Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:24:42.467096725Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=warn ts=2026-05-12T22:24:44.427885386Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:24:45.180365545Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:24:45.884599501Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-12T22:24:47.392632907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="119 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:24:47.402551596Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:24:49.302205673Z caller=checkpoint.go:372 msg="writing series" size="438 kB"
level=warn ts=2026-05-12T22:24:49.428169535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:24:50.111874741Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T22:24:52.126600869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:39042"
level=debug ts=2026-05-12T22:24:52.72077439Z caller=checkpoint.go:372 msg="writing series" size="899 kB"
level=debug ts=2026-05-12T22:24:52.720795315Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=6
level=warn ts=2026-05-12T22:24:54.428256661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:24:56.138515411Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=info ts=2026-05-12T22:24:56.395531265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="50 MB" avg_comp="2.5 MB" total_uncomp="414 MB" avg_uncomp="21 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:24:56.403546692Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-12T22:24:59.42722459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:24:59.555591315Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-12T22:25:00.097562493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:25:00.106319765Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:25:02.972578679Z caller=checkpoint.go:372 msg="writing series" size="4.0 kB"
level=warn ts=2026-05-12T22:25:04.427925276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:25:06.391119707Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=info ts=2026-05-12T22:25:09.427855498Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:25:09.427893102Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:25:09.427904142Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:25:09.427946327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:25:09.80832759Z caller=checkpoint.go:372 msg="writing series" size="732 kB"
level=debug ts=2026-05-12T22:25:09.808347613Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=debug ts=2026-05-12T22:25:13.226894003Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-12T22:25:13.226915329Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=warn ts=2026-05-12T22:25:14.427429591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:25:14.791772304Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:25:16.643731067Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=debug ts=2026-05-12T22:25:18.552697001Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:25:19.427220523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:25:20.061794885Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
ts=2026-05-12T22:25:20.114336088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=info ts=2026-05-12T22:25:21.790016936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:25:21.794546786Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:25:23.479687075Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-12T22:25:24.428198045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:25:26.89746979Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
ts=2026-05-12T22:25:28.073017836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:42592"
level=warn ts=2026-05-12T22:25:29.427765625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:25:30.314944177Z caller=checkpoint.go:372 msg="writing series" size="217 kB"
level=debug ts=2026-05-12T22:25:33.732363313Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-12T22:25:34.427527409Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:25:36.05086831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="119 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:25:36.066078705Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:25:37.15016097Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=debug ts=2026-05-12T22:25:39.057558338Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:25:39.410940979Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:25:39.410983562Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:25:39.427337792Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:25:39.427356297Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:25:39.427363773Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:25:39.427384915Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:25:40.568400313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:25:40.568485767Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=error ts=2026-05-12T22:25:40.57715231Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:25:40.577173842Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:25:40.577197497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:25:40.584082294Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:25:43.985401554Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=info ts=2026-05-12T22:25:44.336787689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="50 MB" avg_comp="2.5 MB" total_uncomp="414 MB" avg_uncomp="21 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:25:44.345622346Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1e17b1d1%3A19e1e282a93%3Aefa29a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-12T22:25:44.345642021Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:25:44.34566784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:25:44.351135475Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-12T22:25:44.428224487Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:25:47.403695221Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=debug ts=2026-05-12T22:25:47.403716865Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-05-12T22:25:49.428083612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:25:50.11714127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:25:50.822399785Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-12T22:25:50.822421018Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=1
level=info ts=2026-05-12T22:25:52.087635052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:25:52.094942434Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:25:54.239466403Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T22:25:54.239495897Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
level=warn ts=2026-05-12T22:25:54.427818598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:25:57.656812667Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-05-12T22:25:59.428137584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:26:00.42733828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:26:00.432860672Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-12T22:26:01.075104443Z caller=checkpoint.go:372 msg="writing series" size="792 kB"
level=info ts=2026-05-12T22:26:02.49812176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:26:02.502080869Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T22:26:04.428195706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:26:04.492849025Z caller=checkpoint.go:372 msg="writing series" size="466 kB"
level=debug ts=2026-05-12T22:26:04.492873159Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=debug ts=2026-05-12T22:26:04.569893525Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:26:07.910528082Z caller=checkpoint.go:372 msg="writing series" size="278 kB"
level=debug ts=2026-05-12T22:26:08.270450604Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:26:09.427770425Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:26:09.427802177Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:26:09.427810161Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:26:09.427851167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:26:11.327085953Z caller=checkpoint.go:372 msg="writing series" size="267 kB"
level=warn ts=2026-05-12T22:26:14.427993105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:26:14.744683157Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-12T22:26:16.175387979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="119 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:26:16.188095418Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:26:18.162719619Z caller=checkpoint.go:372 msg="writing series" size="261 kB"
level=warn ts=2026-05-12T22:26:19.427643719Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:26:20.119943654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:26:21.58083347Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=info ts=2026-05-12T22:26:21.951042121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:26:21.958089121Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-12T22:26:22.136023073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:57248"
level=warn ts=2026-05-12T22:26:24.427723456Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:26:24.998593771Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=debug ts=2026-05-12T22:26:24.99861262Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-12T22:26:28.208376622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:26:28.215028059Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-12T22:26:28.417422066Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T22:26:28.41744327Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
level=warn ts=2026-05-12T22:26:29.427312026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:26:31.833775214Z caller=checkpoint.go:372 msg="writing series" size="729 kB"
level=warn ts=2026-05-12T22:26:34.427458799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:26:35.252096978Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=debug ts=2026-05-12T22:26:35.252122666Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=debug ts=2026-05-12T22:26:38.669648454Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-12T22:26:39.411763504Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:26:39.412935757Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:26:39.427176912Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:26:39.427212956Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:26:39.427220695Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:26:39.427243832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:26:42.087664824Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=debug ts=2026-05-12T22:26:43.998615976Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:26:44.427359018Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:26:45.504453353Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=info ts=2026-05-12T22:26:46.682313739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:26:46.688171155Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:26:48.924521233Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-12T22:26:48.924548171Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=4
level=warn ts=2026-05-12T22:26:49.427815353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:26:50.121710602Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T22:26:52.138283957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:60792"
level=debug ts=2026-05-12T22:26:52.343116585Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=debug ts=2026-05-12T22:26:53.060564948Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:26:53.062345937Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T22:26:54.427224719Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:26:55.75802585Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
ts=2026-05-12T22:26:58.079117854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:48732"
level=debug ts=2026-05-12T22:26:59.175940059Z caller=checkpoint.go:372 msg="writing series" size="282 kB"
level=warn ts=2026-05-12T22:26:59.427457889Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:27:00.421705173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="119 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:27:00.431981409Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:27:02.592976182Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-12T22:27:04.428173825Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:27:06.0110607Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-12T22:27:09.428064727Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:27:09.428089158Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:27:09.428096699Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:27:09.42811798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:27:09.429182933Z caller=checkpoint.go:384 msg="flushing series" totalSize="359 kB" series=5
level=info ts=2026-05-12T22:27:09.431580102Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-12T22:27:09.432750742Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.004622912s
level=warn ts=2026-05-12T22:27:14.427680232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:27:15.088018784Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:27:19.427625505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:27:20.124355855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=info ts=2026-05-12T22:27:21.529801809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:27:21.540642147Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:27:24.427535031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:27:27.668179434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:27:27.674148929Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-12T22:27:29.427353389Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:27:32.550960578Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:27:34.427905925Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:27:39.410826313Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:27:39.410973978Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:27:39.427669756Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-12T22:27:39.42768152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:27:39.427706985Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:27:39.427725439Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:27:39.427733248Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T22:27:39.427810395Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-12T22:27:39.427821064Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T22:27:39.427838899Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.586µs success=true
level=debug ts=2026-05-12T22:27:39.428075618Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=info ts=2026-05-12T22:27:40.43389803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:27:40.439224539Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:27:41.501472602Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:27:42.76212261Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-12T22:27:44.427915269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:27:46.095181681Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-12T22:27:47.539348468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="119 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:27:47.547910028Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:27:47.547940421Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:27:47.547972891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="119 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:27:47.556069987Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:27:49.428148428Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=warn ts=2026-05-12T22:27:49.428154661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:27:50.127535432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T22:27:51.163957818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:41226"
ts=2026-05-12T22:27:52.143183287Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:35736"
level=debug ts=2026-05-12T22:27:52.761586703Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-12T22:27:54.428055568Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:27:56.095552108Z caller=checkpoint.go:372 msg="writing series" size="487 kB"
level=warn ts=2026-05-12T22:27:59.427892232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:27:59.429197505Z caller=checkpoint.go:372 msg="writing series" size="528 kB"
level=debug ts=2026-05-12T22:27:59.429213919Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=info ts=2026-05-12T22:27:59.468052136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="119 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:27:59.478968126Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:28:02.761949026Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=warn ts=2026-05-12T22:28:04.427767049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:28:06.095167233Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=info ts=2026-05-12T22:28:09.427235876Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:28:09.427246943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:28:09.42726207Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:28:09.427270606Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:28:09.428443969Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=debug ts=2026-05-12T22:28:12.761810585Z caller=checkpoint.go:372 msg="writing series" size="658 kB"
level=debug ts=2026-05-12T22:28:12.761830997Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-12T22:28:12.876875182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:28:12.885593709Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:28:14.427823514Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:28:14.764873668Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:28:16.095214285Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=debug ts=2026-05-12T22:28:16.143491238Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:28:16.143540195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:28:16.148888917Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:28:16.880084863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:28:16.885041882Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-12T22:28:19.427976232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:28:19.428024531Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
ts=2026-05-12T22:28:20.130813589Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=info ts=2026-05-12T22:28:21.035884515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="119 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:28:21.043034146Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:28:22.76244202Z caller=checkpoint.go:372 msg="writing series" size="618 kB"
level=warn ts=2026-05-12T22:28:24.427670625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:28:26.094970389Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-12T22:28:28.854126215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:28:28.861349354Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-12T22:28:29.427941572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:28:29.483929139Z caller=checkpoint.go:372 msg="writing series" size="55 MB"
level=debug ts=2026-05-12T22:28:29.483955327Z caller=checkpoint.go:384 msg="flushing series" totalSize="56 MB" series=5
level=info ts=2026-05-12T22:28:32.479265244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:28:32.485271743Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:28:32.762004519Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-12T22:28:34.427425699Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:28:36.095467607Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-05-12T22:28:39.410898481Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:28:39.411228882Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:28:39.427485417Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:28:39.427488329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:28:39.427498932Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:28:39.427507292Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:28:39.428759382Z caller=checkpoint.go:372 msg="writing series" size="225 kB"
level=debug ts=2026-05-12T22:28:41.220185929Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:28:41.220182392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" idle=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:28:41.223373257Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-12T22:28:41.521032694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="122 B" avg_comp="122 B" total_uncomp="47 kB" avg_uncomp="47 kB" idle=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:28:41.52382444Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e16aa0a%3A19e1e176532%3Aa06c984a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:28:41.820702154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="141 B" avg_comp="141 B" total_uncomp="30 kB" avg_uncomp="30 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:28:41.823674491Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:28:42.120502745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:28:42.123315355Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:28:42.763537168Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-12T22:28:42.76355879Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=4
level=warn ts=2026-05-12T22:28:44.427421649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:28:46.09777096Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-12T22:28:46.097802963Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=1
level=warn ts=2026-05-12T22:28:49.427754252Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:28:49.428780034Z caller=checkpoint.go:372 msg="writing series" size="77 kB"
ts=2026-05-12T22:28:50.134018356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T22:28:51.168364315Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:54456"
level=info ts=2026-05-12T22:28:51.43581038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:28:51.44200361Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-12T22:28:52.147961722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:44642"
level=info ts=2026-05-12T22:28:52.181507325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:28:52.184723012Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-12T22:28:52.762389662Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-12T22:28:54.42733649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:28:55.10765985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:28:55.111160216Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T22:28:56.10910694Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-12T22:28:56.109128634Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=3
level=info ts=2026-05-12T22:28:56.124872609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:28:56.128237902Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e16aa0a%3A19e1e176532%3Aa06c984a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:28:56.423197852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:28:56.426014924Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-12T22:28:58.086274454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:41284"
level=debug ts=2026-05-12T22:28:59.428206436Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=warn ts=2026-05-12T22:28:59.428215469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:29:02.762093727Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=warn ts=2026-05-12T22:29:04.427966771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:29:06.097515177Z caller=checkpoint.go:372 msg="writing series" size="3.4 MB"
level=debug ts=2026-05-12T22:29:06.097541184Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.6 MB" series=3
level=debug ts=2026-05-12T22:29:07.958974187Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:29:08.951718301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:29:08.960287101Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:29:09.427784509Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:29:09.427808145Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:29:09.427815296Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:29:09.427810561Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:29:09.429061999Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-12T22:29:12.744478242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="119 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:29:12.751616065Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:29:12.761607984Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-12T22:29:13.315353853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" idle=1 labels="{stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:29:13.315379381Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-12T22:29:13.32163996Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:29:13.61605452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="192 B" avg_comp="192 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:29:13.619068576Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:29:13.915910422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:29:13.918777147Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-12T22:29:14.427368602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:29:15.65387829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:29:15.659101928Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-12T22:29:16.09581891Z caller=checkpoint.go:372 msg="writing series" size="229 kB"
level=info ts=2026-05-12T22:29:18.134332942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:29:18.137371588Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-12T22:29:19.427409529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:29:19.428526383Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
ts=2026-05-12T22:29:20.137367498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=debug ts=2026-05-12T22:29:22.761905994Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=info ts=2026-05-12T22:29:22.896205864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:29:22.899323247Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-12T22:29:24.427677999Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:29:26.095193741Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-12T22:29:27.640352097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:29:27.643288956Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:29:27.93704241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:29:27.940450241Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-12T22:29:28.088957818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:59260"
level=info ts=2026-05-12T22:29:28.380212071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:29:28.383639626Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:29:29.168006824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:29:29.170659529Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-12T22:29:29.427871369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:29:29.428981609Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-12T22:29:30.639262054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:29:30.64254121Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e16aa0a%3A19e1e176532%3Aa06c984a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:29:30.791767325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:29:30.79536551Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:29:30.795387645Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:29:30.79541175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:29:30.798519136Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:29:32.764708726Z caller=checkpoint.go:372 msg="writing series" size="528 kB"
level=debug ts=2026-05-12T22:29:32.764732266Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=warn ts=2026-05-12T22:29:34.427738721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:29:36.095426917Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-12T22:29:37.561719302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:29:37.567913988Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:29:38.4235737Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:29:39.41085012Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:29:39.41145751Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:29:39.427736453Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:29:39.427752481Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:29:39.427760464Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:29:39.427793297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:29:39.428894715Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=debug ts=2026-05-12T22:29:42.761978007Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=warn ts=2026-05-12T22:29:44.427714272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:29:46.095192992Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-12T22:29:49.336617668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:29:49.342898804Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T22:29:49.427521356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:29:49.428796608Z caller=checkpoint.go:372 msg="writing series" size="463 kB"
ts=2026-05-12T22:29:50.140019732Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T22:29:51.177470824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:50686"
level=debug ts=2026-05-12T22:29:51.577153756Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:29:52.152663905Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:37534"
level=debug ts=2026-05-12T22:29:52.762165778Z caller=checkpoint.go:372 msg="writing series" size="899 kB"
level=debug ts=2026-05-12T22:29:52.762189187Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=6
level=warn ts=2026-05-12T22:29:54.428130442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:29:56.09491944Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=info ts=2026-05-12T22:29:56.201010219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:29:56.203796936Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:29:58.37069634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="119 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:29:58.377331318Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:29:59.428155607Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:29:59.428155979Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-12T22:30:00.491338891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:30:00.496425846Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-12T22:30:00.929556889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:30:00.932124236Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:30:01.021496053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:30:01.024150176Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:30:02.762403254Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=info ts=2026-05-12T22:30:04.385957817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:30:04.392032886Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:30:04.427703274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:30:06.095425677Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=info ts=2026-05-12T22:30:06.467086101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:30:06.471707369Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T22:30:08.522203436Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:30:08.522255624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="153 B" avg_comp="153 B" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:30:08.525324747Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:30:09.428205302Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:30:09.428233889Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:30:09.428244051Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:30:09.428244494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:30:09.428735071Z caller=checkpoint.go:372 msg="writing series" size="880 kB"
level=debug ts=2026-05-12T22:30:09.428751105Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=debug ts=2026-05-12T22:30:12.762021571Z caller=checkpoint.go:372 msg="writing series" size="897 kB"
level=warn ts=2026-05-12T22:30:14.428097436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:30:16.095775674Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=debug ts=2026-05-12T22:30:16.095797446Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=info ts=2026-05-12T22:30:17.533177946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:30:17.536518589Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T22:30:19.427572919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:30:19.428665684Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-12T22:30:19.57410761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:30:19.577061867Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-05-12T22:30:20.142594095Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:30:21.180950774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:47276"
level=info ts=2026-05-12T22:30:21.761324834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:30:21.76523023Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:30:22.761786813Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-12T22:30:24.428214532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:30:26.095438242Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=info ts=2026-05-12T22:30:26.88260793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:30:26.885972985Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-12T22:30:28.094717623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:40996"
level=info ts=2026-05-12T22:30:28.385724186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:30:28.392266664Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-12T22:30:29.42754346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:30:29.428970379Z caller=checkpoint.go:372 msg="writing series" size="222 kB"
level=info ts=2026-05-12T22:30:30.082591912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:30:30.085450579Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e16aa0a%3A19e1e176532%3Aa06c984a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-12T22:30:32.761797657Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=debug ts=2026-05-12T22:30:33.97969224Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:30:34.427916622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:30:36.095457821Z caller=checkpoint.go:372 msg="writing series" size="309 kB"
level=debug ts=2026-05-12T22:30:38.92194622Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:30:38.923430106Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-12T22:30:39.410843025Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:30:39.411573679Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=debug ts=2026-05-12T22:30:39.412815778Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-12T22:30:39.412856356Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976249 n=1
level=debug ts=2026-05-12T22:30:39.412896039Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976249 n=1
level=info ts=2026-05-12T22:30:39.42782015Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:30:39.427836616Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:30:39.427845569Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:30:39.427849964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:30:39.429084123Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=info ts=2026-05-12T22:30:40.436217382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:30:40.440708724Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T22:30:42.761851458Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=info ts=2026-05-12T22:30:43.411680926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="119 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:30:43.41833277Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:30:44.427422556Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:30:46.095594357Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=debug ts=2026-05-12T22:30:46.095614929Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-05-12T22:30:48.131292741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:30:48.134019891Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T22:30:49.427244069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:30:49.428422893Z caller=checkpoint.go:372 msg="writing series" size="140 kB"
ts=2026-05-12T22:30:50.145710883Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=info ts=2026-05-12T22:30:52.260857191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:30:52.263655412Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:30:52.764015159Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-12T22:30:52.764038856Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=2
level=info ts=2026-05-12T22:30:53.926572377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:30:53.932278506Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-12T22:30:54.091223441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:30:54.094530835Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-12T22:30:54.427349984Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:30:55.184620417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:30:55.187735292Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:30:56.096628779Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-12T22:30:56.096650325Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=1
level=warn ts=2026-05-12T22:30:59.427718715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:30:59.428844158Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=debug ts=2026-05-12T22:31:02.761792635Z caller=checkpoint.go:372 msg="writing series" size="792 kB"
level=info ts=2026-05-12T22:31:02.98705056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:31:02.996249756Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:31:03.44871919Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:31:04.427601727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:31:06.095487536Z caller=checkpoint.go:372 msg="writing series" size="452 kB"
level=debug ts=2026-05-12T22:31:06.095508483Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=info ts=2026-05-12T22:31:06.810649353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:31:06.815367904Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:31:09.427975033Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:31:09.428001293Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:31:09.428009217Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:31:09.428017672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:31:09.428083292Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=debug ts=2026-05-12T22:31:12.762452006Z caller=checkpoint.go:372 msg="writing series" size="267 kB"
level=warn ts=2026-05-12T22:31:14.428069248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:31:14.847325835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:31:14.850592824Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-12T22:31:16.094977858Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-12T22:31:16.309502493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:31:16.313200217Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:31:17.924137891Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:31:19.42725803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:31:19.428620211Z caller=checkpoint.go:372 msg="writing series" size="310 kB"
ts=2026-05-12T22:31:20.148583853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:31:22.762154523Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=warn ts=2026-05-12T22:31:24.427818355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:31:25.100868084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:31:25.107041963Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:31:26.095729734Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=info ts=2026-05-12T22:31:27.066622496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:31:27.069433123Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e16aa0a%3A19e1e176532%3Aa06c984a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-12T22:31:29.4281162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:31:29.428942056Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T22:31:29.428956244Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=7
level=debug ts=2026-05-12T22:31:32.762570956Z caller=checkpoint.go:372 msg="writing series" size="700 kB"
level=debug ts=2026-05-12T22:31:33.764622883Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:31:33.764681696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="158 kB" avg_uncomp="158 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:31:33.768476195Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-12T22:31:34.427465983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:31:36.095237007Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=debug ts=2026-05-12T22:31:36.09525848Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-05-12T22:31:36.375684626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:31:36.378867234Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:31:36.502338284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:31:36.504792865Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:31:37.223776455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:31:37.230346033Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-12T22:31:39.145727401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="119 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:31:39.153249035Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:31:39.410487072Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:31:39.410865364Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:31:39.427183941Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:31:39.42720357Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:31:39.427212959Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:31:39.427226752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:31:39.428319506Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=debug ts=2026-05-12T22:31:42.762246473Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=info ts=2026-05-12T22:31:43.946230152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:31:43.949329057Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:31:44.252886054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:31:44.256182164Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-12T22:31:44.428132376Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:31:46.095765086Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=info ts=2026-05-12T22:31:47.333344907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:31:47.336699591Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-12T22:31:49.428149092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:31:49.429488321Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-12T22:31:49.429505384Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=4
ts=2026-05-12T22:31:50.151350942Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=info ts=2026-05-12T22:31:50.831128278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:31:50.834168817Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T22:31:52.762033764Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-05-12T22:31:52.834496622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:31:52.838024386Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T22:31:54.427411415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:31:56.095469721Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-12T22:31:57.306483567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:31:57.312819122Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-12T22:31:58.100850088Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:47342"
level=warn ts=2026-05-12T22:31:59.428128885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:31:59.428191222Z caller=checkpoint.go:372 msg="writing series" size="282 kB"
level=debug ts=2026-05-12T22:32:02.468453359Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:32:02.76215775Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-12T22:32:04.428054917Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:32:06.095108926Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-12T22:32:06.141341224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:32:06.144095475Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-12T22:32:07.564779451Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:32:09.427694942Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:32:09.427725037Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:32:09.427734042Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:32:09.427737076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:32:09.428811365Z caller=checkpoint.go:384 msg="flushing series" totalSize="360 kB" series=5
level=info ts=2026-05-12T22:32:09.435989052Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000011.tmp new=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-12T22:32:09.437836571Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009854521s
level=info ts=2026-05-12T22:32:11.499752789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:32:11.503941671Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T22:32:14.427984663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:32:16.327002307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:32:16.332338403Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-12T22:32:19.427925332Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:32:20.153823093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=info ts=2026-05-12T22:32:20.698594739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:32:20.703453084Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e16aa0a%3A19e1e176532%3Aa06c984a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-12T22:32:21.76732735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:32:21.770085568Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-12T22:32:22.167941628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:51830"
level=info ts=2026-05-12T22:32:23.169349296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:32:23.174275824Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:32:23.174311731Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:32:23.174336744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:32:23.178577889Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-12T22:32:24.427557647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:32:24.44968545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:32:24.452326659Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:32:29.09718421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:32:29.100400898Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-12T22:32:29.428097527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:32:31.999511854Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:32:32.857089397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="516 B" avg_comp="516 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:32:32.860178188Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:32:34.307918233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="119 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:32:34.314486639Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:32:34.427347798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:32:34.609505391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:32:34.612181932Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:32:35.428170702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:32:35.431371918Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:32:36.37761842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:32:36.380319414Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:32:38.950982023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:32:38.958210634Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-12T22:32:39.410869774Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:32:39.411242561Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:32:39.427646851Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:32:39.427665207Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:32:39.427661599Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-12T22:32:39.427672847Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:32:39.427689133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:32:39.427765902Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-12T22:32:39.427773898Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T22:32:39.427789024Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=10.675µs success=true
level=debug ts=2026-05-12T22:32:39.428044249Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=debug ts=2026-05-12T22:32:42.761789428Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-12T22:32:44.42815861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:32:45.036345008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:32:45.043057963Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:32:45.043077772Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:32:45.043102452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:32:45.049217422Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:32:46.095456667Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-12T22:32:49.427940814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:32:49.427983869Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
ts=2026-05-12T22:32:50.156669741Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:32:52.76147691Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-12T22:32:53.801886625Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:32:54.427487959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:32:56.052861438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:32:56.058870092Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:32:56.095057554Z caller=checkpoint.go:372 msg="writing series" size="378 kB"
level=info ts=2026-05-12T22:32:57.501430357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:32:57.508392272Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:32:59.428043174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:32:59.428184511Z caller=checkpoint.go:372 msg="writing series" size="425 kB"
level=debug ts=2026-05-12T22:32:59.428200188Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-12T22:32:59.605218278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:32:59.608053772Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-12T22:33:02.166695231Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:33:02.761675196Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=warn ts=2026-05-12T22:33:04.427658871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:33:06.094724389Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=warn ts=2026-05-12T22:33:09.427322913Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:33:09.427328716Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:33:09.427349947Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:33:09.427359218Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:33:09.428512399Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=info ts=2026-05-12T22:33:09.69958382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:33:09.702924851Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:33:10.151430747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:33:10.157756553Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:33:11.188586547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:33:11.191119801Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T22:33:12.762028101Z caller=checkpoint.go:372 msg="writing series" size="593 kB"
level=debug ts=2026-05-12T22:33:12.762050139Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-12T22:33:14.048416779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:33:14.051561035Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-12T22:33:14.42772912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:33:14.927722534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:33:14.932790114Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-12T22:33:16.094956357Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=info ts=2026-05-12T22:33:18.847696294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="119 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:33:18.854224373Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:33:19.039156426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:33:19.04176245Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:33:19.136969932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:33:19.139666331Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-12T22:33:19.427214191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:33:19.428255536Z caller=checkpoint.go:372 msg="writing series" size="1.0 kB"
level=info ts=2026-05-12T22:33:19.570590704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:33:19.573336597Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e16aa0a%3A19e1e176532%3Aa06c984a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-05-12T22:33:20.159810637Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:33:22.76254426Z caller=checkpoint.go:372 msg="writing series" size="470 kB"
level=warn ts=2026-05-12T22:33:24.428090292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:33:26.095371185Z caller=checkpoint.go:372 msg="writing series" size="6.2 kB"
level=info ts=2026-05-12T22:33:26.499743225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="684 B" avg_comp="684 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:33:26.509128973Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:33:26.53416573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:33:26.541162249Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:33:29.427230414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:33:29.480994499Z caller=checkpoint.go:372 msg="writing series" size="55 MB"
level=debug ts=2026-05-12T22:33:29.481020521Z caller=checkpoint.go:384 msg="flushing series" totalSize="56 MB" series=5
level=debug ts=2026-05-12T22:33:31.902309576Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:33:31.902365731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:33:31.90596125Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T22:33:32.762343933Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-12T22:33:32.766426062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:33:32.769371051Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T22:33:34.427432374Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:33:36.094745534Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-05-12T22:33:36.974465929Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:33:39.410840972Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:33:39.41101277Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=warn ts=2026-05-12T22:33:39.427332766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:33:39.427337811Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:33:39.427357231Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:33:39.427365569Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:33:39.428569576Z caller=checkpoint.go:372 msg="writing series" size="225 kB"
level=debug ts=2026-05-12T22:33:42.763578218Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-12T22:33:42.76360324Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=4
level=warn ts=2026-05-12T22:33:44.427343466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:33:44.703360911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:33:44.71047268Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T22:33:46.096627643Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-12T22:33:46.0966486Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=1
level=warn ts=2026-05-12T22:33:49.427989016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:33:49.428059232Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
ts=2026-05-12T22:33:50.163276238Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=info ts=2026-05-12T22:33:50.280141061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:33:50.284052644Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:33:50.536726585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:33:50.540135249Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-12T22:33:52.761447652Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-12T22:33:54.428215364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:33:54.66512967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:33:54.668696457Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T22:33:56.109358724Z caller=checkpoint.go:372 msg="writing series" size="17 MB"
level=debug ts=2026-05-12T22:33:56.109379906Z caller=checkpoint.go:384 msg="flushing series" totalSize="17 MB" series=3
level=info ts=2026-05-12T22:33:57.128563737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:33:57.131872945Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-12T22:33:58.111366516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:57196"
level=warn ts=2026-05-12T22:33:59.427237228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:33:59.428334909Z caller=checkpoint.go:372 msg="writing series" size="1.6 kB"
level=debug ts=2026-05-12T22:33:59.597796517Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:34:02.305874025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:34:02.312434628Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:34:02.761771608Z caller=checkpoint.go:372 msg="writing series" size="229 kB"
level=warn ts=2026-05-12T22:34:04.427934068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:34:06.071567615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:34:06.074865275Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e16aa0a%3A19e1e176532%3Aa06c984a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-12T22:34:06.09695226Z caller=checkpoint.go:372 msg="writing series" size="3.3 MB"
level=debug ts=2026-05-12T22:34:06.096981953Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.5 MB" series=3
level=info ts=2026-05-12T22:34:08.828350182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:34:08.83301963Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:34:08.834067938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:34:08.840476689Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:34:09.427737408Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:34:09.427764557Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:34:09.427772803Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:34:09.427847217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:34:09.429001577Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-12T22:34:10.827494047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:34:10.832454453Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-12T22:34:11.796785819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:34:11.799502679Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:34:11.809824499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:34:11.812190829Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T22:34:12.761684026Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-12T22:34:13.458998708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:34:13.461584885Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T22:34:14.427262754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:34:15.493672935Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:34:15.49552745Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-12T22:34:16.095162299Z caller=checkpoint.go:372 msg="writing series" size="303 kB"
level=info ts=2026-05-12T22:34:17.901592572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:34:17.904593567Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:34:17.987722953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="119 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:34:17.99399537Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:34:17.994014002Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:34:17.994039717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="119 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:34:18.000102751Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:34:19.428181532Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=warn ts=2026-05-12T22:34:19.428195266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:34:20.166768384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:34:22.179115991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:53566"
level=debug ts=2026-05-12T22:34:22.76170324Z caller=checkpoint.go:372 msg="writing series" size="213 kB"
level=info ts=2026-05-12T22:34:23.246850403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:34:23.250689977Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-12T22:34:24.42815955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:34:26.094701721Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-12T22:34:29.427547935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:34:29.428587156Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-12T22:34:29.80066944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="119 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:34:29.808812922Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:34:32.76262873Z caller=checkpoint.go:372 msg="writing series" size="528 kB"
level=debug ts=2026-05-12T22:34:32.762651225Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=debug ts=2026-05-12T22:34:33.972738768Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:34:34.427853212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:34:36.095235801Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-12T22:34:38.215281031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:34:38.218243757Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:34:38.86505329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:34:38.868393683Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:34:39.410813174Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:34:39.411603228Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=warn ts=2026-05-12T22:34:39.427917541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:34:39.42793311Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:34:39.427943297Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=debug ts=2026-05-12T22:34:39.427952168Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:34:39.42795976Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:34:42.761714673Z caller=checkpoint.go:372 msg="writing series" size="33 kB"
level=warn ts=2026-05-12T22:34:44.427975037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:34:45.421342268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:34:45.428162249Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:34:45.428186877Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:34:45.428216699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:34:45.43486492Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:34:46.09473654Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-12T22:34:48.016899787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:34:48.021180238Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T22:34:49.427631681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:34:49.42902912Z caller=checkpoint.go:372 msg="writing series" size="473 kB"
level=info ts=2026-05-12T22:34:49.806705638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:34:49.809425425Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-05-12T22:34:50.170109235Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:34:51.207143567Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:57448"
level=debug ts=2026-05-12T22:34:52.762510154Z caller=checkpoint.go:372 msg="writing series" size="899 kB"
level=debug ts=2026-05-12T22:34:52.762531958Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=warn ts=2026-05-12T22:34:54.42803392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:34:55.039521094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:34:55.042558022Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:34:55.042577636Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:34:55.042599744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:34:55.045963663Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:34:55.705506385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="119 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:34:55.713119777Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:34:56.095393195Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=info ts=2026-05-12T22:34:57.460353921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:34:57.463223094Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e16aa0a%3A19e1e176532%3Aa06c984a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-05-12T22:34:58.115642Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:55970"
level=info ts=2026-05-12T22:34:58.226540579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:34:58.229769503Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:34:58.553675879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:34:58.556471871Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-12T22:34:59.42764275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:34:59.428646255Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-12T22:34:59.939289072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:34:59.94576757Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-12T22:35:00.063207491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:35:00.069113158Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:35:00.250059242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:35:00.252677311Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:35:00.252696342Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:35:00.252718057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:35:00.255138747Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:35:00.996051706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:35:01.002566841Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:35:02.761827804Z caller=checkpoint.go:372 msg="writing series" size="4.5 kB"
level=warn ts=2026-05-12T22:35:04.427704138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:35:06.095506905Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=info ts=2026-05-12T22:35:06.245039827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:35:06.248764577Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T22:35:07.951342028Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:35:09.427464286Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:35:09.427489259Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:35:09.427497461Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:35:09.427501991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:35:09.429525199Z caller=checkpoint.go:372 msg="writing series" size="972 kB"
level=debug ts=2026-05-12T22:35:09.429539115Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=5
level=info ts=2026-05-12T22:35:09.46273227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:35:09.465487554Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:35:09.727135201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:35:09.729860533Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T22:35:12.360392164Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:35:12.36539444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:35:12.367897976Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:35:12.762361503Z caller=checkpoint.go:372 msg="writing series" size="898 kB"
level=warn ts=2026-05-12T22:35:14.427256148Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:35:16.095434264Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=debug ts=2026-05-12T22:35:16.095457005Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=warn ts=2026-05-12T22:35:19.427210807Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:35:19.428240634Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
ts=2026-05-12T22:35:20.17291881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=debug ts=2026-05-12T22:35:22.76145352Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-12T22:35:24.421276201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=warn ts=2026-05-12T22:35:24.427403529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-12T22:35:24.435666327Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:35:26.095434495Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=info ts=2026-05-12T22:35:28.506671193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:35:28.511417081Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T22:35:29.427347049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:35:29.428606948Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=debug ts=2026-05-12T22:35:32.762077362Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=warn ts=2026-05-12T22:35:34.427398837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:35:34.800291859Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:35:35.39532502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:35:35.399443458Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T22:35:36.095015846Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=info ts=2026-05-12T22:35:38.528520003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:35:38.532135316Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:35:38.532163746Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:35:38.532189366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:35:38.534541579Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:35:39.410831475Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:35:39.411551521Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:35:39.427827095Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:35:39.427845246Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:35:39.427853409Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:35:39.427875597Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:35:39.429081481Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=info ts=2026-05-12T22:35:39.49933812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:35:39.502203725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:35:39.50314936Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:35:39.503150949Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:35:42.761572525Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=warn ts=2026-05-12T22:35:44.428061587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:35:46.095769588Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=info ts=2026-05-12T22:35:47.569777983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:35:47.576175977Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-12T22:35:48.340063054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:35:48.343134091Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:35:48.343153261Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:35:48.343174379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:35:48.346381792Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:35:48.616713611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:35:48.619888674Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-05-12T22:35:48.619906999Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:35:48.644075811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:35:48.646572045Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-12T22:35:48.702661794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:35:48.705322979Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1e16aa0a%3A19e1e176532%3Aa06c984a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-12T22:35:48.705336436Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:35:48.943759661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="525 kB" avg_uncomp="525 kB" idle=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:35:48.946857384Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-12T22:35:49.427226524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:35:49.428392236Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-05-12T22:35:49.473347186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:35:49.480019626Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-12T22:35:50.175373329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T22:35:51.211453769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:50402"
ts=2026-05-12T22:35:52.185825696Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:42986"
level=info ts=2026-05-12T22:35:52.266770681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="119 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:35:52.274720089Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:35:52.763804642Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-12T22:35:52.763826623Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=11
level=info ts=2026-05-12T22:35:54.277495187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:35:54.280567328Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-12T22:35:54.427812738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:35:55.342663219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:35:55.34692619Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-12T22:35:55.346945512Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:35:55.346967403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:35:55.349231856Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T22:35:56.09624077Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-12T22:35:56.096267814Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=1
level=debug ts=2026-05-12T22:35:58.954607558Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:35:59.230457594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:35:59.233320965Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-12T22:35:59.427756884Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:35:59.42891632Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-12T22:36:00.503975717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:36:00.507456815Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T22:36:01.273422765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:36:01.276261858Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:36:01.327476477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:36:01.330201539Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-12T22:36:02.762903055Z caller=checkpoint.go:372 msg="writing series" size="792 kB"
level=info ts=2026-05-12T22:36:04.34105753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:36:04.343935081Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-12T22:36:04.428015304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:36:06.094994889Z caller=checkpoint.go:372 msg="writing series" size="452 kB"
level=debug ts=2026-05-12T22:36:06.095016419Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=warn ts=2026-05-12T22:36:09.42795574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:36:09.427971388Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:36:09.427991432Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:36:09.427999324Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:36:09.428145687Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=debug ts=2026-05-12T22:36:10.095800997Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:36:12.761666711Z caller=checkpoint.go:372 msg="writing series" size="267 kB"
level=warn ts=2026-05-12T22:36:14.427714125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:36:14.660902423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:36:14.665962705Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:36:15.178807949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:36:15.182331067Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:36:15.182360437Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:36:15.182382286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:36:15.185325116Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T22:36:16.094891914Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-12T22:36:19.428078805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:36:19.428442457Z caller=checkpoint.go:372 msg="writing series" size="310 kB"
ts=2026-05-12T22:36:20.17809677Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=info ts=2026-05-12T22:36:21.604497416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:36:21.60883465Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:36:21.608855591Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:36:21.608877957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:36:21.612611319Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:36:22.762091901Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=info ts=2026-05-12T22:36:23.362586739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:36:23.368153641Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-12T22:36:24.428034292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:36:26.094985803Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=info ts=2026-05-12T22:36:27.33263994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:36:27.33658691Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-12T22:36:28.120808527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:47344"
level=warn ts=2026-05-12T22:36:29.427794567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:36:29.43043531Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T22:36:29.430450671Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=7
level=info ts=2026-05-12T22:36:31.290084194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:36:31.292885026Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T22:36:32.762400799Z caller=checkpoint.go:372 msg="writing series" size="700 kB"
level=info ts=2026-05-12T22:36:32.784926891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:36:32.788856973Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:36:33.349485997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:36:33.352489641Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:36:33.612481814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="119 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:36:33.619090534Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:36:34.427834205Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:36:35.446996222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:36:35.449707742Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:36:36.005960729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:36:36.008711273Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:36:36.09512892Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=debug ts=2026-05-12T22:36:36.095146754Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-05-12T22:36:36.292357997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:36:36.295422841Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:36:37.164472901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:36:37.167150646Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-12T22:36:39.403255631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:36:39.406070546Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-12T22:36:39.410859232Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:36:39.411153304Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:36:39.427412453Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:36:39.427434382Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:36:39.427449653Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:36:39.427447847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:36:39.428556731Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-12T22:36:40.004294385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:36:40.006949643Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:36:40.756967629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:36:40.762279942Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-12T22:36:42.222498572Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:36:42.762221722Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=warn ts=2026-05-12T22:36:44.427713424Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:36:45.692894004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:36:45.700796007Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:36:46.095033556Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=warn ts=2026-05-12T22:36:49.428052004Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:36:49.431908086Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-12T22:36:49.431928316Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=4
level=info ts=2026-05-12T22:36:49.549541753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:36:49.5527743Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-05-12T22:36:50.180862681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:36:50.236842174Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:36:52.761613859Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-05-12T22:36:52.901064979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:36:52.904596722Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:36:54.235483762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:36:54.238270549Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T22:36:54.427591788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:36:56.094709981Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-12T22:36:59.427212206Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:36:59.428352358Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=debug ts=2026-05-12T22:37:02.761386643Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-12T22:37:04.42735755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:37:06.095181989Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-12T22:37:07.308774891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:37:07.317740582Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:37:08.919322212Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:37:09.428047577Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:37:09.428072948Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:37:09.428080969Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:37:09.428094162Z caller=checkpoint.go:384 msg="flushing series" totalSize="141 kB" series=5
level=warn ts=2026-05-12T22:37:09.428099889Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:37:09.435022808Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000012.tmp new=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-12T22:37:09.43624084Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008341028s
level=warn ts=2026-05-12T22:37:14.427525338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:37:17.873611108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:37:17.87936373Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T22:37:19.427631468Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:37:20.183052204Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=info ts=2026-05-12T22:37:21.761774486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:37:21.765445473Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-05-12T22:37:22.19345795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:50646"
level=warn ts=2026-05-12T22:37:24.428168731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:37:24.680177535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:37:24.683652109Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-12T22:37:29.427313135Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:37:30.334272755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:37:30.337737022Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:37:31.906505971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:37:31.909696779Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-12T22:37:32.09515272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="140 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:37:32.10168094Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:37:32.586091559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:37:32.589228649Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:37:32.629553062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:37:32.632047214Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:37:33.332784622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="516 B" avg_comp="516 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:37:33.335918747Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-12T22:37:34.427654Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:37:35.043803805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:37:35.051131995Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-12T22:37:37.399418798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:37:37.405629709Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:37:39.410701351Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:37:39.411057075Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:37:39.427413867Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-12T22:37:39.427422958Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:37:39.427435303Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-12T22:37:39.4274276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:37:39.427442736Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T22:37:39.427545027Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-12T22:37:39.427552115Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T22:37:39.427568158Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.366µs success=true
level=debug ts=2026-05-12T22:37:39.427830094Z caller=checkpoint.go:372 msg="writing series" size="225 kB"
level=info ts=2026-05-12T22:37:41.666779048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:37:41.670637076Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:37:42.761690364Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=debug ts=2026-05-12T22:37:44.318141467Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:37:44.427963904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:37:46.094699452Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-12T22:37:46.624482199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:37:46.628185408Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-12T22:37:49.428025869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:37:49.428067455Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=debug ts=2026-05-12T22:37:49.920456018Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:37:50.185429572Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:37:52.762067121Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-12T22:37:53.603489471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:37:53.606440479Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T22:37:54.427240783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:37:56.09514865Z caller=checkpoint.go:372 msg="writing series" size="521 kB"
level=warn ts=2026-05-12T22:37:59.427403826Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:37:59.428822386Z caller=checkpoint.go:372 msg="writing series" size="583 kB"
level=debug ts=2026-05-12T22:37:59.428836482Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=7
level=debug ts=2026-05-12T22:38:02.761448044Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=info ts=2026-05-12T22:38:02.973879131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:38:02.980430358Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-12T22:38:04.427742771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:38:05.97523037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:38:05.979370319Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T22:38:06.094769166Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=info ts=2026-05-12T22:38:09.428055306Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:38:09.428082081Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:38:09.428090355Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:38:09.428095155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:38:09.428179534Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=debug ts=2026-05-12T22:38:12.761720066Z caller=checkpoint.go:372 msg="writing series" size="593 kB"
level=debug ts=2026-05-12T22:38:12.761740412Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-12T22:38:13.657874998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:38:13.661049854Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-12T22:38:14.427662143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:38:16.030012048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:38:16.035143112Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-12T22:38:16.095310155Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=debug ts=2026-05-12T22:38:16.335993203Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:38:19.428189027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:38:19.428203963Z caller=checkpoint.go:372 msg="writing series" size="1.0 kB"
ts=2026-05-12T22:38:20.187347436Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=info ts=2026-05-12T22:38:21.421282146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:38:21.426018297Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:38:21.429064086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:38:21.431879474Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-12T22:38:22.761751287Z caller=checkpoint.go:372 msg="writing series" size="512 kB"
level=warn ts=2026-05-12T22:38:24.427413141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:38:24.962980153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="684 B" avg_comp="684 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:38:24.965928903Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:38:26.094961121Z caller=checkpoint.go:372 msg="writing series" size="6.2 kB"
level=info ts=2026-05-12T22:38:26.340442485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:38:26.347748086Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:38:28.799626146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:38:28.802993176Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:38:28.803015303Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:38:28.803040063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:38:28.805334024Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-12T22:38:29.427998762Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:38:29.486067552Z caller=checkpoint.go:372 msg="writing series" size="55 MB"
level=debug ts=2026-05-12T22:38:29.486093248Z caller=checkpoint.go:384 msg="flushing series" totalSize="55 MB" series=5
level=info ts=2026-05-12T22:38:30.356474311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="140 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:38:30.363739033Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:38:30.391780308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:38:30.394232224Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:38:30.472979704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:38:30.475626929Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-12T22:38:32.761529923Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=debug ts=2026-05-12T22:38:34.318681717Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:38:34.320231893Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T22:38:34.427908434Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:38:36.094719456Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-05-12T22:38:38.392784865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:38:38.407139991Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:38:39.234419653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:38:39.237446294Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:38:39.41081937Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:38:39.411004746Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=warn ts=2026-05-12T22:38:39.427309002Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:38:39.427321317Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:38:39.427344519Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:38:39.427351673Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:38:39.428768654Z caller=checkpoint.go:372 msg="writing series" size="225 kB"
level=info ts=2026-05-12T22:38:39.860115384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:38:39.863658344Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T22:38:42.76217574Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-12T22:38:42.762197871Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=4
level=warn ts=2026-05-12T22:38:44.428051159Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:38:45.050077124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:38:45.056446856Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:38:46.097242608Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-12T22:38:46.097263923Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=1
level=warn ts=2026-05-12T22:38:49.427756947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:38:49.427789593Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
ts=2026-05-12T22:38:50.189101873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:38:50.430394173Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:38:51.570728855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:38:51.575726709Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T22:38:52.761118016Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-12T22:38:53.314747981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:38:53.317832081Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T22:38:54.427442229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:38:56.113720715Z caller=checkpoint.go:372 msg="writing series" size="18 MB"
level=debug ts=2026-05-12T22:38:56.113745297Z caller=checkpoint.go:384 msg="flushing series" totalSize="18 MB" series=3
ts=2026-05-12T22:38:58.129647538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:41020"
level=debug ts=2026-05-12T22:38:59.428046322Z caller=checkpoint.go:372 msg="writing series" size="1.6 kB"
level=warn ts=2026-05-12T22:38:59.428093213Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:38:59.934672353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:38:59.939066914Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T22:39:02.761754507Z caller=checkpoint.go:372 msg="writing series" size="286 kB"
level=info ts=2026-05-12T22:39:02.854883558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:39:02.861276962Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:39:02.861316081Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:39:02.861338242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:39:02.866676859Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-12T22:39:04.427243949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:39:06.098211523Z caller=checkpoint.go:372 msg="writing series" size="3.3 MB"
level=debug ts=2026-05-12T22:39:06.098232604Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.6 MB" series=3
level=info ts=2026-05-12T22:39:09.427379933Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:39:09.427417002Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-12T22:39:09.427420122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:39:09.427428191Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:39:09.42878137Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-12T22:39:10.010088217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:39:10.013085171Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:39:10.420325381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:39:10.423369691Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-12T22:39:11.401974752Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:39:12.217403254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="140 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:39:12.223845939Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:39:12.762142046Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-12T22:39:12.97667681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:39:12.979534404Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-12T22:39:14.427257076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:39:16.094858346Z caller=checkpoint.go:372 msg="writing series" size="377 kB"
level=debug ts=2026-05-12T22:39:18.092453322Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:39:19.42798622Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=warn ts=2026-05-12T22:39:19.427990623Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:39:19.846415687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:39:19.849511881Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-05-12T22:39:20.191289581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:39:21.22918186Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:48028"
level=info ts=2026-05-12T22:39:21.3407159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:39:21.346727991Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-12T22:39:22.188933615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:39:22.192569564Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-12T22:39:22.204213649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:34326"
level=debug ts=2026-05-12T22:39:22.761469841Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=warn ts=2026-05-12T22:39:24.427493602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:39:24.990694846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:39:24.997936506Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:39:24.997956172Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:39:24.997977356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:39:25.005662489Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:39:26.094946364Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-12T22:39:27.01334042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:39:27.016344921Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-12T22:39:29.427360727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:39:29.428464846Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-12T22:39:30.752522976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:39:30.756091844Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T22:39:32.761925676Z caller=checkpoint.go:372 msg="writing series" size="528 kB"
level=debug ts=2026-05-12T22:39:32.761948539Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=info ts=2026-05-12T22:39:32.945903733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:39:32.949006153Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:39:33.586783747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:39:33.592317058Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-12T22:39:34.428185746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:39:36.094493721Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-12T22:39:37.724004572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:39:37.727681165Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:39:39.410825456Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:39:39.411205581Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:39:39.427475121Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:39:39.427494273Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:39:39.427504237Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:39:39.427527597Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:39:39.428606886Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-12T22:39:40.463570513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:39:40.473765772Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:39:41.529924924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:39:41.536857677Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:39:42.761260949Z caller=checkpoint.go:372 msg="writing series" size="33 kB"
level=warn ts=2026-05-12T22:39:44.427840097Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:39:44.79590382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:39:44.799126696Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T22:39:46.094816101Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-05-12T22:39:49.42734551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:39:49.428859369Z caller=checkpoint.go:372 msg="writing series" size="473 kB"
level=debug ts=2026-05-12T22:39:50.021599568Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-12T22:39:50.193434266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:39:52.762776299Z caller=checkpoint.go:372 msg="writing series" size="899 kB"
level=debug ts=2026-05-12T22:39:52.762803956Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=warn ts=2026-05-12T22:39:54.427740389Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:39:56.095587538Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=info ts=2026-05-12T22:39:56.818565319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:39:56.823198648Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:39:56.983286714Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:39:58.492682999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="140 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:39:58.500090063Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:39:59.427824Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:39:59.427824976Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-12T22:39:59.610850895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:39:59.616400582Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-12T22:40:02.761920725Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-12T22:40:04.428046312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:40:04.696350985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:40:04.69958506Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T22:40:06.095022208Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=info ts=2026-05-12T22:40:07.243509633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:40:07.246814232Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:40:09.02286294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:40:09.031476634Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:40:09.427379774Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:40:09.427405421Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:40:09.427413568Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:40:09.427417281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:40:09.429609012Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-12T22:40:09.429619753Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=5
level=info ts=2026-05-12T22:40:09.649497026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:40:09.653818228Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-12T22:40:10.330664443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:40:10.333746743Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-12T22:40:11.650496756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:40:11.653517232Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T22:40:12.762071263Z caller=checkpoint.go:372 msg="writing series" size="898 kB"
level=warn ts=2026-05-12T22:40:14.427358444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:40:14.645662217Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:40:16.095562739Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=debug ts=2026-05-12T22:40:16.095586083Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=debug ts=2026-05-12T22:40:19.427798853Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-12T22:40:19.427810738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:40:20.195698287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=info ts=2026-05-12T22:40:22.212602021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:40:22.217357307Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T22:40:22.761751343Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-12T22:40:24.427929665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:40:26.095183783Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-12T22:40:29.427201697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:40:29.428448231Z caller=checkpoint.go:372 msg="writing series" size="230 kB"
level=info ts=2026-05-12T22:40:29.837176718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:40:29.843194344Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-12T22:40:29.843216455Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:40:29.843244188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:40:29.846620666Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T22:40:30.335647407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:40:30.342475837Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:40:32.762005892Z caller=checkpoint.go:372 msg="writing series" size="8.8 kB"
level=info ts=2026-05-12T22:40:33.218419364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:40:33.221382586Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:40:33.266406798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:40:33.269729824Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T22:40:34.427818194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:40:36.094666528Z caller=checkpoint.go:372 msg="writing series" size="172 kB"
level=debug ts=2026-05-12T22:40:37.226895024Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:40:38.872450689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="140 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:40:38.880452769Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:40:38.880474687Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:40:38.88049994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="140 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:40:38.887472816Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:40:39.410828747Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:40:39.411183977Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=warn ts=2026-05-12T22:40:39.427477123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:40:39.427497521Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:40:39.427517659Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:40:39.427524995Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:40:39.42892673Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=info ts=2026-05-12T22:40:41.962667843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:40:41.966717162Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:40:42.215642689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:40:42.218978122Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T22:40:42.761404112Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=warn ts=2026-05-12T22:40:44.427745193Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:40:44.870434962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:40:44.873995101Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:40:46.094990967Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=warn ts=2026-05-12T22:40:49.427607022Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:40:49.428821746Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=debug ts=2026-05-12T22:40:49.428835305Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-05-12T22:40:49.560970633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:40:49.568066076Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:40:49.648749182Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-12T22:40:50.198272354Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:40:51.237708351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:36738"
level=info ts=2026-05-12T22:40:51.556153391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="140 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:40:51.563121561Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-12T22:40:52.211993084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:53790"
level=info ts=2026-05-12T22:40:52.553177207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:40:52.556098875Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-12T22:40:52.76268066Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-12T22:40:52.762699419Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=1
level=info ts=2026-05-12T22:40:54.076957738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:40:54.080368142Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:40:54.290049661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:40:54.295003391Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-12T22:40:54.427368272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:40:56.095897712Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-12T22:40:56.095919731Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=1
ts=2026-05-12T22:40:58.139470362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:55420"
level=warn ts=2026-05-12T22:40:59.427317902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:40:59.428525656Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
level=debug ts=2026-05-12T22:41:02.762461982Z caller=checkpoint.go:372 msg="writing series" size="792 kB"
level=info ts=2026-05-12T22:41:03.043738971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:41:03.046818504Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:41:04.353410321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:41:04.35668193Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-12T22:41:04.427714115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:41:04.907975184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:41:04.9117331Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T22:41:05.948022723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:41:05.950881867Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:41:06.095467311Z caller=checkpoint.go:372 msg="writing series" size="469 kB"
level=debug ts=2026-05-12T22:41:06.095488643Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=info ts=2026-05-12T22:41:09.427781932Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:41:09.427791822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:41:09.427813107Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:41:09.427823231Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:41:09.427893047Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=debug ts=2026-05-12T22:41:12.761846077Z caller=checkpoint.go:372 msg="writing series" size="267 kB"
level=warn ts=2026-05-12T22:41:14.427880991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:41:16.094735534Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-05-12T22:41:17.601810745Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:41:19.427215195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:41:19.428476414Z caller=checkpoint.go:372 msg="writing series" size="310 kB"
level=info ts=2026-05-12T22:41:19.685647738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:41:19.690936686Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:41:19.729395696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:41:19.732195418Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-12T22:41:20.200500507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=info ts=2026-05-12T22:41:20.297832961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="140 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:41:20.304213348Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:41:20.579429005Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:41:22.388514209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:41:22.395030551Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:41:22.39505574Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:41:22.395088172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:41:22.400508074Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:41:22.761779799Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=warn ts=2026-05-12T22:41:24.427556889Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:41:24.600921115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:41:24.603798358Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T22:41:26.094482859Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=warn ts=2026-05-12T22:41:29.427955861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:41:29.429119672Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T22:41:29.429134222Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=7
level=debug ts=2026-05-12T22:41:32.76211957Z caller=checkpoint.go:372 msg="writing series" size="712 kB"
level=warn ts=2026-05-12T22:41:34.42743025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:41:36.09494497Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=debug ts=2026-05-12T22:41:36.094968117Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-05-12T22:41:36.296349902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:41:36.30341917Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:41:36.303440716Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:41:36.303461329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:41:36.30607355Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:41:37.993595929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:41:37.996622713Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:41:38.548421208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:41:38.554570529Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:41:39.41087205Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:41:39.411463562Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=warn ts=2026-05-12T22:41:39.427799319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:41:39.42780492Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:41:39.427809381Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=debug ts=2026-05-12T22:41:39.427822468Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:41:39.427830333Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T22:41:39.919935669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:41:39.925168257Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-12T22:41:42.485686241Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:41:42.761606256Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=info ts=2026-05-12T22:41:43.489692297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:41:43.496315701Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:41:44.427456587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:41:46.07889621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:41:46.0818294Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T22:41:46.094944736Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=info ts=2026-05-12T22:41:47.351781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:41:47.354558936Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-12T22:41:49.427262189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:41:49.430178582Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-12T22:41:49.4301928Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=4
level=info ts=2026-05-12T22:41:50.193812836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:41:50.198367358Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-05-12T22:41:50.202562381Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=info ts=2026-05-12T22:41:51.115416001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="516 B" avg_comp="516 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:41:51.118212021Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:41:51.118231632Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:41:51.118255488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:41:51.120397223Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-12T22:41:51.243036188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:36098"
ts=2026-05-12T22:41:52.216144694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:34564"
level=info ts=2026-05-12T22:41:52.689931775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:41:52.6929093Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T22:41:52.761081824Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=warn ts=2026-05-12T22:41:54.427490369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:41:55.794778038Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:41:55.796812348Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-12T22:41:56.09484284Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=warn ts=2026-05-12T22:41:59.427490051Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:41:59.428595669Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=info ts=2026-05-12T22:41:59.891116406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:41:59.894652925Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T22:42:01.815185682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:42:01.818183046Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-12T22:42:02.761803818Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-12T22:42:04.427440333Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:42:06.094887386Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-12T22:42:08.343986499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:42:08.3469994Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:42:09.427926012Z caller=checkpoint.go:384 msg="flushing series" totalSize="142 kB" series=5
level=info ts=2026-05-12T22:42:09.427946139Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:42:09.427950343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:42:09.427973485Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:42:09.427981782Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T22:42:09.43463574Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000013.tmp new=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-12T22:42:09.436203365Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008516494s
level=info ts=2026-05-12T22:42:11.309030966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:42:11.31221486Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:42:11.41572369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:42:11.418941512Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:42:11.740751319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="140 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:42:11.748478459Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:42:14.427589013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:42:17.712337304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:42:17.718554402Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:42:18.041427642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:42:18.045019823Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:42:18.273698513Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:42:18.27376005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:42:18.279167381Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-12T22:42:19.427760551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:42:20.205507385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T22:42:21.246444205Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:44700"
level=info ts=2026-05-12T22:42:23.554541674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:42:23.562237333Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:42:24.427955527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:42:25.003080332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:42:25.008934902Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-12T22:42:27.38141923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:42:27.384569048Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:42:27.384588527Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:42:27.384610224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:42:27.386972401Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:42:29.016135109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:42:29.01961463Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T22:42:29.427227197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:42:30.550990555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:42:30.553809328Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-12T22:42:30.553830255Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:42:30.553849608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:42:30.556191261Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:42:31.533455561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:42:31.536412805Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:42:31.536430112Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:42:31.536448987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:42:31.538866114Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:42:32.714136538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:42:32.720723675Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-12T22:42:34.427755138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:42:37.728938851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:42:37.731792906Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:42:38.707589772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:42:38.710884769Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:42:39.410886693Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:42:39.411466329Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:42:39.427738199Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:42:39.427749844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:42:39.427758157Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:42:39.427765851Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T22:42:39.427770053Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-12T22:42:39.427896328Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-12T22:42:39.42790473Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T22:42:39.427921831Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.573µs success=true
level=debug ts=2026-05-12T22:42:39.428199993Z caller=checkpoint.go:372 msg="writing series" size="225 kB"
level=debug ts=2026-05-12T22:42:42.761968542Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-05-12T22:42:43.338522987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:42:43.341547857Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-12T22:42:44.427403871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:42:46.095446537Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-12T22:42:46.198733916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:42:46.201566344Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:42:47.961581147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:42:47.964427553Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-05-12T22:42:47.964452976Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:42:47.964476151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:42:47.966931745Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-12T22:42:49.325038358Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:42:49.427807833Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:42:49.428949007Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
ts=2026-05-12T22:42:50.208778475Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T22:42:52.220964551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:41354"
level=debug ts=2026-05-12T22:42:52.761532216Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-12T22:42:53.194699568Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:42:54.427700167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:42:54.98578579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:42:54.989011623Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T22:42:56.095600115Z caller=checkpoint.go:372 msg="writing series" size="422 kB"
level=info ts=2026-05-12T22:42:56.340706519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:42:56.343830606Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:42:56.343849094Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:42:56.343870007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:42:56.346031364Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-12T22:42:58.087577253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:42:58.090970215Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:42:58.090994592Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:42:58.09101759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:42:58.093949905Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-12T22:42:58.148315149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:45164"
level=info ts=2026-05-12T22:42:58.778936804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:42:58.781703065Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-12T22:42:59.427415532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:42:59.428721309Z caller=checkpoint.go:372 msg="writing series" size="502 kB"
level=debug ts=2026-05-12T22:42:59.428736992Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=debug ts=2026-05-12T22:43:02.762570096Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=info ts=2026-05-12T22:43:02.80878631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="140 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:43:02.815349279Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:43:03.30574833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:43:03.308274282Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:43:03.308315615Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:43:03.308335513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:43:03.310416507Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T22:43:04.427744514Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:43:06.095469095Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=info ts=2026-05-12T22:43:07.38313299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:43:07.389888043Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:43:07.727125044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:43:07.73011056Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:43:09.427478592Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:43:09.427505079Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:43:09.427515397Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:43:09.427523354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:43:09.428699865Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=info ts=2026-05-12T22:43:10.160249031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:43:10.164123301Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:43:11.473178993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:43:11.47619478Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:43:11.761979476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:43:11.768209007Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T22:43:12.461262106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:43:12.466117264Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-12T22:43:12.762168732Z caller=checkpoint.go:372 msg="writing series" size="593 kB"
level=debug ts=2026-05-12T22:43:12.762190451Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-12T22:43:13.159869609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:43:13.162757755Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-12T22:43:13.552934855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:43:13.559060417Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:43:14.427416032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:43:16.094924187Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=info ts=2026-05-12T22:43:16.683697478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:43:16.686566552Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-12T22:43:19.427954034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:43:19.429014689Z caller=checkpoint.go:372 msg="writing series" size="1.0 kB"
level=info ts=2026-05-12T22:43:20.049659761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="684 B" avg_comp="684 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:43:20.052535224Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-12T22:43:20.211980023Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:43:22.67172079Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:43:22.762338165Z caller=checkpoint.go:372 msg="writing series" size="527 kB"
level=info ts=2026-05-12T22:43:23.149745363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:43:23.154959485Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T22:43:24.427967648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:43:24.817250621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:43:24.820694202Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:43:26.095521573Z caller=checkpoint.go:372 msg="writing series" size="6.2 kB"
level=info ts=2026-05-12T22:43:26.75243944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:43:26.755486438Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T22:43:29.427226231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:43:29.48018376Z caller=checkpoint.go:372 msg="writing series" size="55 MB"
level=debug ts=2026-05-12T22:43:29.480208528Z caller=checkpoint.go:384 msg="flushing series" totalSize="56 MB" series=5
level=debug ts=2026-05-12T22:43:32.762163008Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-12T22:43:34.428092681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:43:35.579142074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:43:35.582608236Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-12T22:43:36.095238381Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-05-12T22:43:37.792067808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:43:37.795523789Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-12T22:43:39.410874481Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:43:39.411246176Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:43:39.42762572Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:43:39.427630709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:43:39.427640973Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:43:39.427648922Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:43:39.429036061Z caller=checkpoint.go:372 msg="writing series" size="225 kB"
level=info ts=2026-05-12T22:43:41.003111566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:43:41.007066677Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T22:43:42.763662476Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-12T22:43:42.76368515Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=4
level=info ts=2026-05-12T22:43:43.188175787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:43:43.191898328Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T22:43:44.427700385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:43:45.483351512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:43:45.486664204Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T22:43:46.097161893Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-12T22:43:46.097183543Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=1
level=warn ts=2026-05-12T22:43:49.427963169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:43:49.429069592Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
ts=2026-05-12T22:43:50.215009692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T22:43:51.555033338Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:43:52.224972361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:56150"
level=debug ts=2026-05-12T22:43:52.761974724Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-12T22:43:52.804121259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="140 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:43:52.811716006Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:43:54.427818297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:43:56.112762774Z caller=checkpoint.go:372 msg="writing series" size="19 MB"
level=debug ts=2026-05-12T22:43:56.112786958Z caller=checkpoint.go:384 msg="flushing series" totalSize="19 MB" series=3
level=debug ts=2026-05-12T22:43:57.364090872Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:43:58.824351899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:43:58.830817859Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-12T22:43:59.427229125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:43:59.428319798Z caller=checkpoint.go:372 msg="writing series" size="1.6 kB"
level=info ts=2026-05-12T22:44:01.245945992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:44:01.251609876Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-12T22:44:02.404060054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:44:02.407323576Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T22:44:02.76252472Z caller=checkpoint.go:372 msg="writing series" size="332 kB"
level=warn ts=2026-05-12T22:44:04.428141982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:44:04.697084993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:44:04.703036468Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:44:05.055022672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:44:05.057813071Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T22:44:06.098220345Z caller=checkpoint.go:372 msg="writing series" size="3.4 MB"
level=debug ts=2026-05-12T22:44:06.098242499Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.7 MB" series=3
level=info ts=2026-05-12T22:44:06.745470175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:44:06.75337031Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:44:09.427165403Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:44:09.427193133Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:44:09.42720153Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:44:09.42722482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:44:09.42871665Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-12T22:44:09.925326018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:44:09.928261556Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:44:11.441204296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:44:11.444216985Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:44:12.761801175Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-05-12T22:44:14.427359488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:44:15.453119741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:44:15.456250795Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T22:44:16.096002184Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=info ts=2026-05-12T22:44:16.490087348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:44:16.493147307Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-12T22:44:19.428039083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:44:19.429195652Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
ts=2026-05-12T22:44:20.21805183Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T22:44:21.25679281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:36990"
level=debug ts=2026-05-12T22:44:22.762098245Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=info ts=2026-05-12T22:44:23.254349865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:44:23.259282849Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-12T22:44:24.427247281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:44:26.095201892Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-12T22:44:29.396826845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:44:29.402983071Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-12T22:44:29.427711784Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:44:29.42876454Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-12T22:44:31.66311649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:44:31.666386936Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T22:44:31.791342443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:44:31.795128694Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:44:31.918658273Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:44:32.762783345Z caller=checkpoint.go:372 msg="writing series" size="528 kB"
level=debug ts=2026-05-12T22:44:32.762803736Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=debug ts=2026-05-12T22:44:33.945550863Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:44:34.428135025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:44:36.09538261Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-12T22:44:39.009600527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="140 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:44:39.01657073Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:44:39.410806288Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:44:39.410834809Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:44:39.427496635Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:44:39.427515991Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:44:39.427523588Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:44:39.427528328Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:44:39.428626982Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-12T22:44:39.513102507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:44:39.515698076Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T22:44:42.762453039Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=warn ts=2026-05-12T22:44:44.428053257Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:44:45.366231218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:44:45.372086015Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-12T22:44:46.095476979Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-12T22:44:48.355525646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:44:48.361085018Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-12T22:44:49.42824823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:44:49.428982567Z caller=checkpoint.go:372 msg="writing series" size="521 kB"
ts=2026-05-12T22:44:50.221200588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=info ts=2026-05-12T22:44:50.446100717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:44:50.452077112Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-12T22:44:51.261008572Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:41128"
level=debug ts=2026-05-12T22:44:52.763010232Z caller=checkpoint.go:372 msg="writing series" size="899 kB"
level=debug ts=2026-05-12T22:44:52.763031712Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=info ts=2026-05-12T22:44:53.27089998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:44:53.274937707Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-12T22:44:54.427534999Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:44:54.675772055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:44:54.678865754Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:44:54.678886678Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:44:54.67890978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:44:54.681347242Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:44:55.214669979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:44:55.217512994Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:44:56.095982631Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=debug ts=2026-05-12T22:44:58.094564307Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:44:59.42772392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:44:59.428769998Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-12T22:44:59.886737305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:44:59.890662422Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T22:45:02.761575622Z caller=checkpoint.go:372 msg="writing series" size="5.0 kB"
level=info ts=2026-05-12T22:45:03.561173255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:45:03.56816601Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T22:45:04.427746718Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:45:05.571508155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:45:05.575044906Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-12T22:45:06.095618859Z caller=checkpoint.go:372 msg="writing series" size="121 kB"
level=info ts=2026-05-12T22:45:08.340776905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:45:08.343778327Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:45:09.428215308Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:45:09.428240475Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:45:09.428248111Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:45:09.4282644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:45:09.428682291Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-12T22:45:09.428695852Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=5
level=debug ts=2026-05-12T22:45:12.762322779Z caller=checkpoint.go:372 msg="writing series" size="898 kB"
level=info ts=2026-05-12T22:45:13.721738781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:45:13.725123729Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:45:13.749161038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:45:13.752730305Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:45:14.230348321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:45:14.233044061Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T22:45:14.427250716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:45:16.094961877Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=debug ts=2026-05-12T22:45:16.094986825Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=warn ts=2026-05-12T22:45:19.427901644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:45:19.428931358Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
ts=2026-05-12T22:45:20.224636125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T22:45:21.264447846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:52270"
level=debug ts=2026-05-12T22:45:21.390954877Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:45:22.761789925Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-12T22:45:23.30650023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:45:23.310592263Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-12T22:45:23.476051246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="140 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:45:23.483079243Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:45:24.427542995Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:45:24.828102366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:45:24.831565061Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T22:45:26.096871609Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-05-12T22:45:26.10578853Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:45:27.698338038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:45:27.703440228Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:45:27.703463082Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:45:27.703485981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:45:27.70799467Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
ts=2026-05-12T22:45:28.163007049Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:34918"
level=warn ts=2026-05-12T22:45:29.427890562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:45:29.429054077Z caller=checkpoint.go:372 msg="writing series" size="234 kB"
level=info ts=2026-05-12T22:45:31.66825509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:45:31.670968837Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T22:45:32.762057586Z caller=checkpoint.go:372 msg="writing series" size="8.8 kB"
level=warn ts=2026-05-12T22:45:34.428286958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:45:36.095614288Z caller=checkpoint.go:372 msg="writing series" size="266 kB"
level=info ts=2026-05-12T22:45:39.410909195Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:45:39.4109474Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=debug ts=2026-05-12T22:45:39.412168521Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-12T22:45:39.412220357Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976250 n=1
level=debug ts=2026-05-12T22:45:39.412274994Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976250 n=1
level=info ts=2026-05-12T22:45:39.427225984Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:45:39.427242488Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-12T22:45:39.427234702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:45:39.42725045Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:45:39.428492159Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=info ts=2026-05-12T22:45:39.767345923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:45:39.772448566Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T22:45:40.385718735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:45:40.3886928Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:45:42.528413772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:45:42.533695056Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:45:42.762413973Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=info ts=2026-05-12T22:45:43.167414319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:45:43.172462773Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-12T22:45:44.427956187Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:45:45.667494639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:45:45.670660083Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:45:46.095854103Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=debug ts=2026-05-12T22:45:46.095875217Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-05-12T22:45:46.613594194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:45:46.61646281Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-12T22:45:49.427846237Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:45:49.428988844Z caller=checkpoint.go:372 msg="writing series" size="159 kB"
level=info ts=2026-05-12T22:45:50.098911791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:45:50.105765311Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:45:50.105790091Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:45:50.105821203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:45:50.112194239Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-12T22:45:50.228238905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=info ts=2026-05-12T22:45:50.245752611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:45:50.248641086Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-12T22:45:51.267579975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:34166"
ts=2026-05-12T22:45:52.236836993Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:37200"
level=debug ts=2026-05-12T22:45:52.763193014Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-12T22:45:52.763213648Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=2
level=info ts=2026-05-12T22:45:54.400414633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:45:54.404416784Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T22:45:54.427263055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:45:56.097138791Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-12T22:45:56.097159584Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=1
level=debug ts=2026-05-12T22:45:56.776786266Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-12T22:45:58.166383187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:51458"
level=info ts=2026-05-12T22:45:59.16695107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:45:59.169843219Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-12T22:45:59.427272682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:45:59.42847172Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
level=debug ts=2026-05-12T22:46:02.761979984Z caller=checkpoint.go:372 msg="writing series" size="792 kB"
level=debug ts=2026-05-12T22:46:04.141540841Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:46:04.143121181Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T22:46:04.427606492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:46:04.72572298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:46:04.728369259Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-12T22:46:06.095598324Z caller=checkpoint.go:372 msg="writing series" size="480 kB"
level=debug ts=2026-05-12T22:46:06.095621761Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=info ts=2026-05-12T22:46:06.243590871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:46:06.246777301Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:46:06.950562545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="140 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:46:06.95846986Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:46:08.374215962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:46:08.381236877Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:46:09.427767062Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:46:09.427793872Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:46:09.427801958Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:46:09.427821364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:46:09.429084105Z caller=checkpoint.go:372 msg="writing series" size="176 kB"
level=info ts=2026-05-12T22:46:10.859576386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:46:10.862999553Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T22:46:12.761933007Z caller=checkpoint.go:372 msg="writing series" size="267 kB"
level=warn ts=2026-05-12T22:46:14.427717183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:46:14.807737185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:46:14.813812494Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-12T22:46:16.095237495Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-12T22:46:19.42822633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:46:19.428336863Z caller=checkpoint.go:372 msg="writing series" size="289 kB"
ts=2026-05-12T22:46:20.231088762Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=info ts=2026-05-12T22:46:20.350029969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:46:20.354273131Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-05-12T22:46:21.271100282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:38554"
level=debug ts=2026-05-12T22:46:22.762186211Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
level=warn ts=2026-05-12T22:46:24.427420027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:46:25.352816054Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T22:46:26.095391902Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=info ts=2026-05-12T22:46:27.364804769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:46:27.372332157Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T22:46:28.576728998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:46:28.583279278Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-12T22:46:29.427460492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:46:29.430054225Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T22:46:29.430069127Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=7
level=info ts=2026-05-12T22:46:30.385564692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:46:30.38856188Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:46:31.872277835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:46:31.881955854Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:46:32.761963069Z caller=checkpoint.go:372 msg="writing series" size="712 kB"
level=info ts=2026-05-12T22:46:33.806506414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:46:33.809468151Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-12T22:46:34.427832233Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:46:34.502995062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:46:34.505800177Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:46:36.094986307Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=debug ts=2026-05-12T22:46:36.095008699Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-05-12T22:46:36.65059908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:46:36.65378829Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-12T22:46:39.410923094Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:46:39.411320179Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:46:39.427605684Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:46:39.427620347Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:46:39.42762809Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:46:39.427652535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:46:39.428729591Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=debug ts=2026-05-12T22:46:42.762514398Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=warn ts=2026-05-12T22:46:44.428086397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:46:45.779319287Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:46:46.095442315Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=info ts=2026-05-12T22:46:47.49142238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:46:47.497214894Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:46:48.115681669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="516 B" avg_comp="516 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:46:48.119118943Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:46:48.3226996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:46:48.325436296Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-12T22:46:49.427493268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:46:49.431380165Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-12T22:46:49.431398782Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=4
level=info ts=2026-05-12T22:46:49.658712494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:46:49.661294651Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-12T22:46:50.234256291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=info ts=2026-05-12T22:46:51.079908675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="140 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:46:51.089350931Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:46:51.08937457Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:46:51.089400194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="140 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:46:51.095641546Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:46:52.299126708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:46:52.301844892Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T22:46:52.762133275Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=warn ts=2026-05-12T22:46:54.428139082Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:46:56.095172092Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-12T22:46:58.160208623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="198 B" avg_comp="198 B" total_uncomp="615 B" avg_uncomp="615 B" idle=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=debug ts=2026-05-12T22:46:58.16026137Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-12T22:46:58.163547761Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
ts=2026-05-12T22:46:58.173153543Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:34392"
level=warn ts=2026-05-12T22:46:59.427724644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:46:59.428795975Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=info ts=2026-05-12T22:47:02.304743861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:47:02.308787486Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-12T22:47:02.762118938Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-12T22:47:04.42794825Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:47:04.832622962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:47:04.837802401Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-12T22:47:06.094905823Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-12T22:47:09.427418562Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:47:09.427426614Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:47:09.427445889Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:47:09.427457106Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:47:09.428524114Z caller=checkpoint.go:384 msg="flushing series" totalSize="132 kB" series=5
level=info ts=2026-05-12T22:47:09.4389751Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000014.tmp new=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-12T22:47:09.440321616Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.012219124s
level=info ts=2026-05-12T22:47:10.569266714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="140 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:47:10.576603335Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:47:11.776710265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T22:47:11.779504509Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=warn ts=2026-05-12T22:47:14.427427874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:47:14.970603428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:47:14.973890675Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-12T22:47:14.973908993Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:47:14.973929219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:47:14.976686901Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T22:47:15.145264948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:47:15.148544012Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:47:16.269933755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:47:16.275575114Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:47:18.251677677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:47:18.256886876Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-12T22:47:19.427531792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:47:20.239078568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:47:22.243382913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:57286"
level=debug ts=2026-05-12T22:47:24.228006487Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:47:24.427445314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:47:26.386960738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:47:26.390183698Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:47:27.400856049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:47:27.404232615Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T22:47:27.489575517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:47:27.497613806Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:47:27.672241051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:47:27.677751389Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T22:47:29.427413746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:47:30.939411933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:47:30.942273794Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-12T22:47:32.038358339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:47:32.041430276Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-12T22:47:34.427787197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:47:34.89133859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:47:34.895803117Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:47:39.410824302Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:47:39.411358553Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:47:39.427675864Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:47:39.427689958Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:47:39.427697823Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T22:47:39.427689303Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-12T22:47:39.427733635Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:47:39.427986599Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-12T22:47:39.42799579Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T22:47:39.428012606Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.676µs success=true
level=debug ts=2026-05-12T22:47:39.428275667Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
level=info ts=2026-05-12T22:47:40.694962844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:47:40.697740497Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:47:41.36859541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T22:47:41.37189218Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=info ts=2026-05-12T22:47:42.006732742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:47:42.009543129Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:47:42.283105339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:47:42.285791527Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T22:47:42.762368363Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-12T22:47:43.407244619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:47:43.410463739Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-12T22:47:44.428218895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:47:45.626101766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="140 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:47:45.632566801Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:47:46.095707792Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-12T22:47:46.680418319Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:47:49.427889211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:47:49.428999488Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
ts=2026-05-12T22:47:50.24239201Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T22:47:52.244508273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:47738"
level=debug ts=2026-05-12T22:47:52.762340952Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=warn ts=2026-05-12T22:47:54.427869593Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:47:56.095484558Z caller=checkpoint.go:372 msg="writing series" size="547 kB"
ts=2026-05-12T22:47:58.176865546Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:57380"
level=warn ts=2026-05-12T22:47:59.42749903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:47:59.429049179Z caller=checkpoint.go:372 msg="writing series" size="660 kB"
level=debug ts=2026-05-12T22:47:59.429063769Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=7
level=debug ts=2026-05-12T22:48:00.129247317Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:48:00.517876721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:48:00.522637788Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:48:02.762069243Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=info ts=2026-05-12T22:48:03.562159357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:48:03.567519026Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-12T22:48:04.427912198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:48:05.215269043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:48:05.220114932Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T22:48:05.298149969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:48:05.30341472Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:48:05.303430763Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:48:05.303451203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:48:05.308436645Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:48:06.095680365Z caller=checkpoint.go:372 msg="writing series" size="894 B"
level=warn ts=2026-05-12T22:48:09.427234619Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:48:09.427240296Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:48:09.427262475Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:48:09.427270919Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:48:09.428469506Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-12T22:48:12.637864204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:48:12.641207582Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T22:48:12.762751312Z caller=checkpoint.go:372 msg="writing series" size="785 kB"
level=debug ts=2026-05-12T22:48:12.762769034Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=warn ts=2026-05-12T22:48:14.427531511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:48:14.932869137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:48:14.937929767Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T22:48:15.691260765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:48:15.694725982Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-12T22:48:16.095493897Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=info ts=2026-05-12T22:48:16.136679899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:48:16.140533006Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:48:17.277653732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:48:17.280484855Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T22:48:19.427579025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:48:19.428637106Z caller=checkpoint.go:372 msg="writing series" size="1.0 kB"
ts=2026-05-12T22:48:20.244386657Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=info ts=2026-05-12T22:48:22.612562139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:48:22.620081317Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:48:22.762736066Z caller=checkpoint.go:372 msg="writing series" size="572 kB"
level=warn ts=2026-05-12T22:48:24.428265039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:48:25.441013664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:48:25.447478029Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:48:25.541753286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:48:25.544360545Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T22:48:26.095872056Z caller=checkpoint.go:372 msg="writing series" size="6.2 kB"
level=debug ts=2026-05-12T22:48:27.72910357Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:48:28.038311698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:48:28.041082239Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:48:28.041102476Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:48:28.04112453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:48:28.043637092Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:48:28.9491435Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:48:29.427236073Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:48:29.478838359Z caller=checkpoint.go:372 msg="writing series" size="55 MB"
level=debug ts=2026-05-12T22:48:29.478860459Z caller=checkpoint.go:384 msg="flushing series" totalSize="56 MB" series=5
level=info ts=2026-05-12T22:48:29.946940696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:48:29.951051076Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:48:29.951076902Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:48:29.951100477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:48:29.95350579Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T22:48:32.761718328Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-12T22:48:33.620820477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:48:33.623821442Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-12T22:48:33.764047535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="140 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:48:33.7728435Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:48:34.427551764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:48:34.806259131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:48:34.809331336Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T22:48:36.095014813Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-05-12T22:48:37.757844012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="626 B" avg_comp="626 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T22:48:37.762534812Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=info ts=2026-05-12T22:48:39.41083234Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:48:39.411606883Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:48:39.427906541Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T22:48:39.427932571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:48:39.427930241Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:48:39.42795295Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:48:39.429135649Z caller=checkpoint.go:372 msg="writing series" size="225 kB"
level=debug ts=2026-05-12T22:48:42.764233934Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-12T22:48:42.764274768Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=4
level=warn ts=2026-05-12T22:48:44.42745142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:48:44.82045864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:48:44.826392389Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-12T22:48:46.097212091Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-12T22:48:46.097234Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=1
level=info ts=2026-05-12T22:48:46.239847621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:48:46.243180398Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-12T22:48:49.427417329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:48:49.427446395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T22:48:49.428368719Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=error ts=2026-05-12T22:48:49.431035818Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-12T22:48:50.246507624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:48:51.283407963Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:51766"
level=debug ts=2026-05-12T22:48:52.762145176Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-12T22:48:54.427432602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:48:55.77942672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:48:55.783571674Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T22:48:56.113097647Z caller=checkpoint.go:372 msg="writing series" size="20 MB"
level=debug ts=2026-05-12T22:48:56.113122827Z caller=checkpoint.go:384 msg="flushing series" totalSize="20 MB" series=3
level=info ts=2026-05-12T22:48:56.638486325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="684 B" avg_comp="684 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:48:56.64169377Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:48:56.641717424Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:48:56.641739802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:48:56.644222782Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:48:57.853587278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:48:57.856757316Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:48:59.073429691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:48:59.079457683Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:48:59.416180607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:48:59.418812617Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T22:48:59.427988513Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:48:59.429028798Z caller=checkpoint.go:372 msg="writing series" size="1.6 kB"
level=info ts=2026-05-12T22:49:01.346848989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:49:01.350459269Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:49:01.350481114Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:49:01.350502819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:49:01.3536076Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:49:02.621655768Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:49:02.762054694Z caller=checkpoint.go:372 msg="writing series" size="159 kB"
level=info ts=2026-05-12T22:49:02.971647302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:49:02.976944204Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:49:02.976962624Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:49:02.976983014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:49:02.981538948Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-12T22:49:03.42914804Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:49:04.427564786Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:49:06.097151889Z caller=checkpoint.go:372 msg="writing series" size="3.3 MB"
level=debug ts=2026-05-12T22:49:06.097176366Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.5 MB" series=3
level=info ts=2026-05-12T22:49:07.725688527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:49:07.728758211Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:49:07.728776606Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:49:07.728799218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:49:07.73253457Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-12T22:49:07.860979583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:49:07.864505101Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:49:09.427184813Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:49:09.427213541Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:49:09.427221195Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:49:09.427228033Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:49:09.42849265Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-12T22:49:12.762434728Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-12T22:49:12.950051508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:49:12.954031823Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:49:13.398092522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:49:13.404004308Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-12T22:49:14.427556746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:49:14.979489159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:49:14.982597141Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-05-12T22:49:14.98261787Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:49:14.982644354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:49:14.985646758Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-12T22:49:15.462208169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:49:15.465096621Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:49:16.095951161Z caller=checkpoint.go:372 msg="writing series" size="304 kB"
level=warn ts=2026-05-12T22:49:19.428265562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:49:19.428273165Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
level=info ts=2026-05-12T22:49:19.726262274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.4 MB" total_uncomp="160 MB" avg_uncomp="20 MB" full=7 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:49:19.734568045Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:49:20.212079696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:49:20.214859403Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-12T22:49:20.250044955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=info ts=2026-05-12T22:49:21.140614519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:49:21.148002343Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-12T22:49:21.286801873Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:44782"
level=debug ts=2026-05-12T22:49:22.762119904Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=warn ts=2026-05-12T22:49:24.427640642Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:49:25.122531611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:49:25.125915223Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:49:25.125939434Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:49:25.125958592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:49:25.128311873Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:49:25.297288219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:49:25.300181715Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-12T22:49:25.300201602Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:49:25.426357817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:49:25.429152Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:49:25.441207144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:49:25.446231277Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-12T22:49:26.078436944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:49:26.081164558Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-12T22:49:26.095333614Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-12T22:49:27.201410965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:49:27.204232828Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-12T22:49:29.427266157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:49:29.428372842Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-12T22:49:29.547666834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T22:49:29.550706211Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=debug ts=2026-05-12T22:49:32.76234898Z caller=checkpoint.go:372 msg="writing series" size="528 kB"
level=debug ts=2026-05-12T22:49:32.762370016Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=info ts=2026-05-12T22:49:34.276000481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:49:34.281044139Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T22:49:34.427418716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:49:36.095152226Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=debug ts=2026-05-12T22:49:38.031956532Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:49:39.410982697Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:49:39.411065895Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:49:39.427322756Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:49:39.427347816Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:49:39.42735756Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:49:39.427359747Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:49:39.428479973Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-12T22:49:40.137350088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:49:40.142367891Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:49:40.772344718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:49:40.775489217Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:49:40.927019696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:49:40.930725466Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T22:49:42.762375775Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=warn ts=2026-05-12T22:49:44.428262503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:49:44.601541108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:49:44.608340296Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:49:45.502752491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:49:45.505788387Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:49:46.09593847Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-12T22:49:46.194222578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:49:46.197164269Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:49:46.197187756Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:49:46.197215005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:49:46.199992587Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T22:49:49.427778275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:49:49.429963533Z caller=checkpoint.go:372 msg="writing series" size="627 kB"
ts=2026-05-12T22:49:50.253650276Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=info ts=2026-05-12T22:49:52.071859065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:49:52.077316763Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-12T22:49:52.762724127Z caller=checkpoint.go:372 msg="writing series" size="899 kB"
level=debug ts=2026-05-12T22:49:52.762744367Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=6
level=info ts=2026-05-12T22:49:52.95487927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:49:52.958110309Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-12T22:49:54.427940142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:49:54.993463207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:49:54.996569847Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-12T22:49:56.095475293Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=info ts=2026-05-12T22:49:56.555694066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:49:56.559622326Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T22:49:56.993239507Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:49:56.994754501Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:49:58.184467805Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:37970"
level=info ts=2026-05-12T22:49:59.338720098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:49:59.341537232Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-12T22:49:59.427364381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:49:59.428450137Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=debug ts=2026-05-12T22:50:02.761831504Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=debug ts=2026-05-12T22:50:03.135008522Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:50:03.533052249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:50:03.536031774Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-12T22:50:04.427817034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:50:06.095437416Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=info ts=2026-05-12T22:50:09.428007762Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:50:09.428034084Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:50:09.428041844Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:50:09.428052452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:50:09.429940187Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-12T22:50:09.429952279Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=5
level=info ts=2026-05-12T22:50:10.704045367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:50:10.710906577Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-12T22:50:12.762151633Z caller=checkpoint.go:372 msg="writing series" size="880 kB"
level=warn ts=2026-05-12T22:50:14.427486929Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:50:16.096070518Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=debug ts=2026-05-12T22:50:16.096094055Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=info ts=2026-05-12T22:50:16.305687811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:50:16.309216877Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:50:16.664424819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.4 MB" total_uncomp="160 MB" avg_uncomp="20 MB" full=7 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:50:16.671416307Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:50:16.706548992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:50:16.709833817Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:50:16.8792494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T22:50:16.881906345Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=info ts=2026-05-12T22:50:18.427476039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:50:18.430346411Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:50:18.451386662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:50:18.456931493Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:50:19.005409164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:50:19.008276986Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-12T22:50:19.427294871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:50:19.4283635Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
ts=2026-05-12T22:50:20.256218909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=info ts=2026-05-12T22:50:20.343157502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:50:20.345859861Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-12T22:50:21.294111816Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:46774"
level=debug ts=2026-05-12T22:50:22.762375048Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=warn ts=2026-05-12T22:50:24.428055513Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:50:26.095655032Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=info ts=2026-05-12T22:50:27.35805526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:50:27.361794124Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-12T22:50:28.187396783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:54846"
level=info ts=2026-05-12T22:50:28.766585685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:50:28.771977657Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-12T22:50:29.427755421Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:50:29.428975695Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=debug ts=2026-05-12T22:50:32.76194395Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=debug ts=2026-05-12T22:50:33.029695685Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:50:33.655055208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:50:33.65836119Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T22:50:34.427430544Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:50:36.096329005Z caller=checkpoint.go:372 msg="writing series" size="336 kB"
level=debug ts=2026-05-12T22:50:37.567343584Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:50:37.567397871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="378 B" avg_comp="378 B" total_uncomp="12 kB" avg_uncomp="12 kB" idle=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T22:50:37.570358781Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=info ts=2026-05-12T22:50:39.152678284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:50:39.156163911Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T22:50:39.41077833Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:50:39.411828266Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:50:39.428085766Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:50:39.42810277Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:50:39.428110849Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:50:39.428107457Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:50:39.429459648Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=debug ts=2026-05-12T22:50:42.761923429Z caller=checkpoint.go:372 msg="writing series" size="234 kB"
level=debug ts=2026-05-12T22:50:42.76194581Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-12T22:50:44.427606814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:50:46.096119358Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=info ts=2026-05-12T22:50:48.878904277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:50:48.882074744Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-12T22:50:49.427310161Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:50:49.428507595Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
level=info ts=2026-05-12T22:50:49.522708474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:50:49.528423387Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T22:50:49.890730461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:50:49.89367724Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-12T22:50:50.256978598Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=info ts=2026-05-12T22:50:50.356939827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T22:50:50.359849249Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=debug ts=2026-05-12T22:50:52.763739805Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-12T22:50:52.763759941Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=3
level=warn ts=2026-05-12T22:50:54.428167665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:50:56.096977402Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-12T22:50:56.096999835Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=1
level=info ts=2026-05-12T22:50:58.7347658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:50:58.739065329Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-12T22:50:59.427322983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:50:59.428487638Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=info ts=2026-05-12T22:51:01.126078563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:51:01.129421337Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T22:51:02.762849346Z caller=checkpoint.go:372 msg="writing series" size="792 kB"
level=info ts=2026-05-12T22:51:03.199622224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:51:03.20326472Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:51:03.333747005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T22:51:03.336941693Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=warn ts=2026-05-12T22:51:04.428036352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:51:05.495049947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:51:05.498900973Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:51:05.801800334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:51:05.806987562Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-12T22:51:06.09574366Z caller=checkpoint.go:372 msg="writing series" size="542 kB"
level=debug ts=2026-05-12T22:51:06.095763907Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=debug ts=2026-05-12T22:51:08.241026609Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:51:08.465668865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:51:08.472637967Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:51:08.903976791Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:51:09.42732392Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:51:09.427353969Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:51:09.427362882Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:51:09.427382279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:51:09.428637906Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=info ts=2026-05-12T22:51:10.130346962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:51:10.13336668Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:51:11.0954249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.4 MB" total_uncomp="160 MB" avg_uncomp="20 MB" full=7 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:51:11.102149744Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:51:12.126105353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:51:12.132543893Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:51:12.762189988Z caller=checkpoint.go:372 msg="writing series" size="267 kB"
level=warn ts=2026-05-12T22:51:14.427812832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:51:14.543127051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:51:14.547992944Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:51:14.981005616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T22:51:14.983559282Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=debug ts=2026-05-12T22:51:16.095811176Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=warn ts=2026-05-12T22:51:19.427204044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:51:19.428356272Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=info ts=2026-05-12T22:51:19.99480665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:51:19.997641975Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:51:19.997659366Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:51:19.997679443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:51:20.000331008Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-12T22:51:20.258594433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:51:22.762235697Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=warn ts=2026-05-12T22:51:24.427449137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:51:26.095606354Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=info ts=2026-05-12T22:51:27.981536472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:51:27.986998596Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-12T22:51:29.427634224Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:51:29.430387365Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T22:51:29.430442703Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=7
level=info ts=2026-05-12T22:51:30.802099484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:51:30.805268448Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T22:51:32.762619894Z caller=checkpoint.go:372 msg="writing series" size="723 kB"
level=warn ts=2026-05-12T22:51:34.42763787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:51:34.702565196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:51:34.705792214Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-12T22:51:35.313609098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:51:35.316822102Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:51:35.977283492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:51:35.979942489Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-12T22:51:36.09574719Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=debug ts=2026-05-12T22:51:36.095781843Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=debug ts=2026-05-12T22:51:39.410657938Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:51:39.410905884Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:51:39.427265686Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:51:39.427285794Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-12T22:51:39.42727977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:51:39.427293254Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:51:39.428423436Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-12T22:51:40.329220203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:51:40.334391917Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-12T22:51:42.762711743Z caller=checkpoint.go:372 msg="writing series" size="178 kB"
level=warn ts=2026-05-12T22:51:44.427538263Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:51:44.452783438Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:51:44.907247191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="516 B" avg_comp="516 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:51:44.91252743Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:51:46.09543238Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=warn ts=2026-05-12T22:51:49.427271328Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:51:49.429746292Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-12T22:51:49.429763317Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=4
level=info ts=2026-05-12T22:51:49.549797875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:51:49.552527102Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-12T22:51:50.261293416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=debug ts=2026-05-12T22:51:51.001090291Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:51:51.301871029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:35306"
level=debug ts=2026-05-12T22:51:52.762199261Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-05-12T22:51:53.899608967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:51:53.905101977Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-12T22:51:54.42757262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:51:54.72158436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:51:54.725247254Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:51:55.896661025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:51:55.899458278Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T22:51:56.095203375Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-12T22:51:59.427804804Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:51:59.428916362Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=info ts=2026-05-12T22:52:01.028000798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:52:01.033704153Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:52:01.033726404Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:52:01.033749176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:52:01.038058475Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-12T22:52:02.087891034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:52:02.091656609Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:52:02.126839279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T22:52:02.129581478Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=debug ts=2026-05-12T22:52:02.762445732Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-12T22:52:04.427899556Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:52:04.927571527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:52:04.93025408Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:52:05.232042196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:52:05.238369833Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:52:05.805244497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T22:52:05.808018243Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=debug ts=2026-05-12T22:52:06.095701437Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=info ts=2026-05-12T22:52:09.316914945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.4 MB" total_uncomp="160 MB" avg_uncomp="20 MB" full=7 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:52:09.323599755Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:52:09.427749773Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:52:09.427776546Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:52:09.427784909Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:52:09.427840055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:52:09.428862559Z caller=checkpoint.go:384 msg="flushing series" totalSize="128 kB" series=5
level=info ts=2026-05-12T22:52:09.436928724Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000015.tmp new=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-12T22:52:09.439004563Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010855349s
level=info ts=2026-05-12T22:52:11.21086526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:52:11.21454436Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T22:52:12.17504667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:52:12.177749845Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T22:52:13.927670197Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:52:14.427748716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:52:17.954728398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:52:17.961692431Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-12T22:52:19.427412833Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:52:19.741464281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:52:19.74436435Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-12T22:52:20.264177038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T22:52:21.304730774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:45900"
level=info ts=2026-05-12T22:52:23.535327883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:52:23.540772027Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-12T22:52:24.428207233Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:52:24.437331475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:52:24.440321887Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T22:52:29.427211696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:52:33.847799937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:52:33.852609134Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:52:34.083015833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:52:34.085706291Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-12T22:52:34.427449913Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:52:37.144764672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:52:37.148050723Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:52:39.410809001Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:52:39.411056476Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:52:39.427309981Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-12T22:52:39.427333394Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:52:39.427355484Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:52:39.427365525Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:52:39.427374153Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:52:39.427476398Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-12T22:52:39.427485186Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T22:52:39.427506305Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.095µs success=true
level=debug ts=2026-05-12T22:52:39.427782608Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=info ts=2026-05-12T22:52:41.272032491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:52:41.279686843Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-12T22:52:41.435417202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:52:41.438861001Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:52:42.761318931Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=info ts=2026-05-12T22:52:44.002763421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:52:44.005625415Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-12T22:52:44.427214754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:52:44.63309152Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:52:46.094654192Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-12T22:52:46.631560175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T22:52:46.634619545Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=info ts=2026-05-12T22:52:46.974339728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T22:52:46.977028124Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=debug ts=2026-05-12T22:52:47.775663029Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:52:48.193724566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:52:48.200616002Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:52:49.42797934Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-12T22:52:49.427987047Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:52:50.267294756Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:52:51.308154853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:53190"
ts=2026-05-12T22:52:52.272500374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:52484"
level=debug ts=2026-05-12T22:52:52.761245284Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-05-12T22:52:54.427346342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:52:56.095108286Z caller=checkpoint.go:372 msg="writing series" size="671 kB"
level=warn ts=2026-05-12T22:52:59.427753522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:52:59.427947771Z caller=checkpoint.go:372 msg="writing series" size="583 kB"
level=debug ts=2026-05-12T22:52:59.427963261Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=7
level=info ts=2026-05-12T22:53:01.080876809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:53:01.085850915Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:53:02.106054225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:53:02.113101247Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:53:02.113120885Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:53:02.113142371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:53:02.118658454Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:53:02.761558911Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=info ts=2026-05-12T22:53:02.829840488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:53:02.832819017Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-12T22:53:04.428204922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:53:04.657065212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.4 MB" total_uncomp="160 MB" avg_uncomp="20 MB" full=7 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:53:04.664883485Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:53:05.079420321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:53:05.089608854Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T22:53:06.094358783Z caller=checkpoint.go:372 msg="writing series" size="894 B"
level=info ts=2026-05-12T22:53:08.295410959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:53:08.29838422Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:53:09.42722327Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:53:09.427252657Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:53:09.427262727Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:53:09.427258807Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:53:09.428780462Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=debug ts=2026-05-12T22:53:12.761950414Z caller=checkpoint.go:372 msg="writing series" size="678 kB"
level=debug ts=2026-05-12T22:53:12.761976376Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-12T22:53:12.778818847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:53:12.784973184Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:53:13.889191508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:53:13.892403528Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-12T22:53:14.427805086Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:53:16.094615192Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=info ts=2026-05-12T22:53:16.389353781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:53:16.392731185Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T22:53:19.427762353Z caller=checkpoint.go:372 msg="writing series" size="1.0 kB"
level=warn ts=2026-05-12T22:53:19.427821942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:53:20.270320407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=debug ts=2026-05-12T22:53:20.485258809Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:53:20.53442337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:53:20.540087879Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-12T22:53:22.761737991Z caller=checkpoint.go:372 msg="writing series" size="730 kB"
level=info ts=2026-05-12T22:53:23.902665736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:53:23.905937387Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:53:24.125263718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:53:24.12829284Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T22:53:24.427637335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:53:26.095030156Z caller=checkpoint.go:372 msg="writing series" size="6.2 kB"
level=debug ts=2026-05-12T22:53:27.726932947Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:53:27.728394168Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T22:53:29.427961024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:53:29.484018447Z caller=checkpoint.go:372 msg="writing series" size="55 MB"
level=debug ts=2026-05-12T22:53:29.484043543Z caller=checkpoint.go:384 msg="flushing series" totalSize="56 MB" series=5
level=info ts=2026-05-12T22:53:29.952168761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:53:29.955440717Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-12T22:53:32.761756172Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-12T22:53:33.632623344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:53:33.636158078Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T22:53:34.42745085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:53:34.961107723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="626 B" avg_comp="626 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T22:53:34.96544882Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T22:53:34.965468404Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:53:34.965489938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T22:53:34.967889734Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=info ts=2026-05-12T22:53:35.445434724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:53:35.451062191Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:53:36.09482697Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-05-12T22:53:36.701533781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:53:36.704286187Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:53:39.173347311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T22:53:39.176575658Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=debug ts=2026-05-12T22:53:39.410534514Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:53:39.410904716Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:53:39.427193006Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:53:39.427212458Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:53:39.427220865Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:53:39.427249049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:53:39.428451904Z caller=checkpoint.go:372 msg="writing series" size="225 kB"
level=info ts=2026-05-12T22:53:39.61148152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:53:39.616766799Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-12T22:53:42.76340124Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-12T22:53:42.763429392Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=4
level=info ts=2026-05-12T22:53:43.883505248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:53:43.88787557Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-12T22:53:44.427245303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:53:46.096987308Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-12T22:53:46.097008514Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=1
level=info ts=2026-05-12T22:53:47.422828612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:53:47.430142381Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:53:48.918243388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T22:53:48.921467667Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=warn ts=2026-05-12T22:53:49.427988246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:53:49.427998593Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
ts=2026-05-12T22:53:50.273895798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T22:53:51.312262638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:40096"
ts=2026-05-12T22:53:52.276582376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:33220"
level=debug ts=2026-05-12T22:53:52.761519133Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-12T22:53:53.845579587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:53:53.850172992Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-12T22:53:53.850195041Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:53:53.850218027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:53:53.854518795Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-12T22:53:54.428247794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:53:55.902885145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:53:55.905676599Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T22:53:56.121672306Z caller=checkpoint.go:372 msg="writing series" size="21 MB"
level=debug ts=2026-05-12T22:53:56.121696384Z caller=checkpoint.go:384 msg="flushing series" totalSize="21 MB" series=3
level=debug ts=2026-05-12T22:53:56.405681551Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:53:57.325343531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.4 MB" total_uncomp="160 MB" avg_uncomp="20 MB" full=7 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:53:57.332711851Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:53:57.332738395Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:53:57.332768682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.5 MB" total_uncomp="160 MB" avg_uncomp="20 MB" full=7 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:53:57.339711513Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T22:53:59.427514677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:53:59.428602685Z caller=checkpoint.go:372 msg="writing series" size="1.6 kB"
level=info ts=2026-05-12T22:53:59.670611462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:53:59.673803206Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-12T22:54:01.571345212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="684 B" avg_comp="684 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:54:01.574376398Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:54:02.762998439Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=warn ts=2026-05-12T22:54:04.428242336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:54:06.045415451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:54:06.048467859Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:54:06.097735587Z caller=checkpoint.go:372 msg="writing series" size="3.4 MB"
level=debug ts=2026-05-12T22:54:06.097760133Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.6 MB" series=3
level=debug ts=2026-05-12T22:54:06.174250502Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:54:07.296190178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:54:07.301446025Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T22:54:08.008800458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:54:08.011580848Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:54:09.427326024Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:54:09.427352498Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:54:09.427361277Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:54:09.427409996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:54:09.428677904Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-12T22:54:10.888478966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:54:10.892874413Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T22:54:12.762056071Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-12T22:54:13.306772295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.5 MB" total_uncomp="160 MB" avg_uncomp="20 MB" full=7 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:54:13.314182202Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:54:13.482843391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:54:13.486157004Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-12T22:54:14.42797138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:54:16.095683773Z caller=checkpoint.go:372 msg="writing series" size="416 kB"
level=info ts=2026-05-12T22:54:16.775052078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T22:54:16.778699632Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=warn ts=2026-05-12T22:54:19.427208922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:54:19.428350576Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
ts=2026-05-12T22:54:20.276592676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T22:54:20.618930745Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:54:22.539439371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:54:22.542614414Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:54:22.55963248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:54:22.565158621Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:54:22.565174209Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:54:22.565194268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:54:22.571755865Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:54:22.761410898Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=info ts=2026-05-12T22:54:22.853829085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:54:22.856600316Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-12T22:54:24.427215926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:54:26.095193051Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-12T22:54:27.569418927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:54:27.574050655Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:54:29.075500382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:54:29.08222147Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:54:29.249784801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:54:29.253087595Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T22:54:29.42822771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:54:29.42827019Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-12T22:54:32.761849599Z caller=checkpoint.go:372 msg="writing series" size="528 kB"
level=debug ts=2026-05-12T22:54:32.761871041Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=warn ts=2026-05-12T22:54:34.428169679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:54:35.233774689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T22:54:35.236875673Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=info ts=2026-05-12T22:54:35.237863769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:54:35.244105301Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:54:36.095141863Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-12T22:54:36.891351807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:54:36.896536913Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-12T22:54:39.410809241Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:54:39.410878754Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:54:39.427169404Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:54:39.427185765Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:54:39.42719315Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:54:39.427204798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:54:39.428331789Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-12T22:54:39.430389209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:54:39.433016052Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T22:54:42.761090134Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-12T22:54:44.427571855Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:54:46.095203148Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=debug ts=2026-05-12T22:54:46.551698479Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:54:49.013520868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:54:49.017233406Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T22:54:49.101335297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.5 MB" total_uncomp="160 MB" avg_uncomp="20 MB" full=7 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:54:49.108203281Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:54:49.223593934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:54:49.226733539Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:54:49.226753326Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:54:49.226775889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:54:49.229142999Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-12T22:54:49.427359742Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:54:49.428792703Z caller=checkpoint.go:372 msg="writing series" size="504 kB"
ts=2026-05-12T22:54:50.2788714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=info ts=2026-05-12T22:54:52.235856888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:54:52.238704197Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-12T22:54:52.281002057Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:57878"
level=debug ts=2026-05-12T22:54:52.762516674Z caller=checkpoint.go:372 msg="writing series" size="899 kB"
level=debug ts=2026-05-12T22:54:52.762538779Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=info ts=2026-05-12T22:54:53.549745219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:54:53.552485805Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:54:53.552508156Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:54:53.5525361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:54:53.555001931Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T22:54:54.427468649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:54:55.067519658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:54:55.070190367Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:54:55.25583605Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:54:56.095125441Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=warn ts=2026-05-12T22:54:59.427544905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:54:59.428719619Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-12T22:55:01.298871326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:55:01.302575648Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:55:02.761851768Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-12T22:55:03.607362202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:55:03.613185111Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-12T22:55:04.427887062Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:55:05.702330557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:55:05.705293174Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:55:05.998912121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T22:55:06.001880724Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=debug ts=2026-05-12T22:55:06.095362059Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-05-12T22:55:08.344322134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:55:08.347114857Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:55:09.427191131Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:55:09.427219334Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-12T22:55:09.427215454Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:55:09.427227123Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:55:09.429694836Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-12T22:55:09.429710236Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=5
level=info ts=2026-05-12T22:55:10.38580881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:55:10.389602637Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-12T22:55:10.621707557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:55:10.624466937Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:55:12.023345906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:55:12.029707973Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:55:12.762068638Z caller=checkpoint.go:372 msg="writing series" size="940 kB"
level=warn ts=2026-05-12T22:55:14.428014044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:55:15.648408715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:55:15.652800078Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:55:15.652817546Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:55:15.652838889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:55:15.656055221Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:55:16.09528037Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=debug ts=2026-05-12T22:55:16.095317874Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=debug ts=2026-05-12T22:55:16.319972803Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:55:16.685919274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:55:16.693199674Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:55:18.382542012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T22:55:18.385987689Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=warn ts=2026-05-12T22:55:19.42765604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:55:19.427668364Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
ts=2026-05-12T22:55:20.282348369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=info ts=2026-05-12T22:55:22.109613833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:55:22.112779617Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-12T22:55:22.281802131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:49478"
level=debug ts=2026-05-12T22:55:22.761863521Z caller=checkpoint.go:372 msg="writing series" size="9.4 kB"
level=info ts=2026-05-12T22:55:23.088872888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:55:23.09182545Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-12T22:55:24.428130088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:55:25.776622527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:55:25.780453523Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T22:55:26.094452855Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
level=info ts=2026-05-12T22:55:28.246627685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:55:28.24925234Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T22:55:29.427277075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:55:29.428650785Z caller=checkpoint.go:372 msg="writing series" size="241 kB"
level=info ts=2026-05-12T22:55:30.060155105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:55:30.065929126Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-12T22:55:32.761926206Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=warn ts=2026-05-12T22:55:34.427669148Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:55:36.095323858Z caller=checkpoint.go:372 msg="writing series" size="364 kB"
level=info ts=2026-05-12T22:55:36.252539282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:55:36.256340076Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T22:55:37.753566077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:55:37.757022695Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T22:55:39.410681665Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:55:39.411727378Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:55:39.42801749Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:55:39.42803277Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:55:39.428040181Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:55:39.428084616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:55:39.428224584Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=debug ts=2026-05-12T22:55:42.761351374Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=debug ts=2026-05-12T22:55:42.761376553Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-05-12T22:55:43.01811949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:55:43.021470433Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:55:43.021491385Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:55:43.021511543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:55:43.024054262Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-12T22:55:44.427721637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:55:46.095044482Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=debug ts=2026-05-12T22:55:46.1933553Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:55:46.193358661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9be421cde0f9c653 immediate=false num_chunks=1 total_comp="174 kB" avg_comp="174 kB" total_uncomp="897 kB" avg_uncomp="897 kB" idle=1 labels="{stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T22:55:46.198003375Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9be421cde0f9c653/19e1e285607%3A19e1e3051c9%3A2a80abf6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-felc\"}"
level=info ts=2026-05-12T22:55:46.479006599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:55:46.482202849Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:55:46.482223372Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:55:46.494402229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:55:46.496801829Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-12T22:55:47.137410168Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:55:48.081952946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.5 MB" total_uncomp="160 MB" avg_uncomp="20 MB" full=7 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:55:48.089405016Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:55:49.365041069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:55:49.370973045Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T22:55:49.427900831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:55:49.428022752Z caller=checkpoint.go:372 msg="writing series" size="173 kB"
ts=2026-05-12T22:55:50.285589625Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=info ts=2026-05-12T22:55:51.806475937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T22:55:51.809367667Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=debug ts=2026-05-12T22:55:52.763051505Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-12T22:55:52.763072627Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=3
level=info ts=2026-05-12T22:55:54.069019907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:55:54.072124728Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-12T22:55:54.072145042Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:55:54.072164501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:55:54.074548045Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-12T22:55:54.427424457Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:55:56.096354565Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-12T22:55:56.096376521Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=1
level=info ts=2026-05-12T22:55:57.515709602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9be421cde0f9c653 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="897 kB" avg_uncomp="897 kB" full=1 labels="{stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T22:55:57.518922373Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9be421cde0f9c653/19e1e285607%3A19e1e3051c9%3A2a80abf6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-felc\"}"
level=info ts=2026-05-12T22:55:58.507693353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:55:58.510562945Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-05-12T22:55:58.510582565Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:55:58.510604712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:55:58.513229248Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-12T22:55:59.168778498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:55:59.171576967Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-12T22:55:59.428075937Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:55:59.428363595Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=info ts=2026-05-12T22:56:00.965104971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:56:00.972185602Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:56:01.227622328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:56:01.233418977Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:56:01.233446308Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:56:01.233470698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:56:01.23856013Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T22:56:01.933616443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:56:01.936247133Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:56:02.131865833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:56:02.134683827Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-12T22:56:02.384333519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:56:02.390249181Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:56:02.761787431Z caller=checkpoint.go:372 msg="writing series" size="792 kB"
level=warn ts=2026-05-12T22:56:04.427875652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:56:06.095691647Z caller=checkpoint.go:372 msg="writing series" size="537 kB"
level=debug ts=2026-05-12T22:56:06.095713347Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=info ts=2026-05-12T22:56:09.428118568Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:56:09.428145843Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:56:09.428153934Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:56:09.428148358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:56:09.42831537Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-12T22:56:09.591650895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T22:56:09.595666898Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=info ts=2026-05-12T22:56:09.754502718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:56:09.75722816Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-12T22:56:11.661477485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:56:11.664608648Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:56:11.664629246Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:56:11.664648986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:56:11.666974279Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:56:12.521404703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:56:12.524018944Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:56:12.692430214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:56:12.694968377Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T22:56:12.761267886Z caller=checkpoint.go:372 msg="writing series" size="267 kB"
level=info ts=2026-05-12T22:56:13.663520008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:56:13.668591295Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-12T22:56:14.427753006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:56:16.094882093Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=debug ts=2026-05-12T22:56:19.134688236Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:56:19.428189022Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:56:19.42828123Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
ts=2026-05-12T22:56:20.287815018Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=info ts=2026-05-12T22:56:20.743586048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:56:20.748581352Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T22:56:22.072100073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:56:22.07508893Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-12T22:56:22.28601343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:59568"
level=debug ts=2026-05-12T22:56:22.761853982Z caller=checkpoint.go:372 msg="writing series" size="160 kB"
level=info ts=2026-05-12T22:56:22.847139178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9be421cde0f9c653 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="897 kB" avg_uncomp="897 kB" full=1 labels="{stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T22:56:22.850665318Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9be421cde0f9c653/19e1e285607%3A19e1e3051c9%3A2a80abf6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-felc\"}"
level=info ts=2026-05-12T22:56:23.342842628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:56:23.345709257Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-12T22:56:24.428019081Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:56:25.498273294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:56:25.500950539Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T22:56:26.095254882Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=info ts=2026-05-12T22:56:27.796961905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:56:27.800043753Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T22:56:29.427283525Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:56:29.429420698Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T22:56:29.429433548Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=7
level=debug ts=2026-05-12T22:56:32.762099042Z caller=checkpoint.go:372 msg="writing series" size="727 kB"
level=warn ts=2026-05-12T22:56:34.428332177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:56:34.763813883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:56:34.76693869Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T22:56:36.095135036Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=debug ts=2026-05-12T22:56:36.095158621Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=info ts=2026-05-12T22:56:36.307784334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:56:36.311607887Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T22:56:37.991935881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:56:37.99491166Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T22:56:38.753207294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.5 MB" total_uncomp="160 MB" avg_uncomp="20 MB" full=7 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:56:38.759698756Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:56:39.410837431Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:56:39.410844347Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:56:39.427487943Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:56:39.427507447Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:56:39.427515858Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:56:39.427535782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:56:39.428575701Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=debug ts=2026-05-12T22:56:42.762038249Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=debug ts=2026-05-12T22:56:44.072084883Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T22:56:44.427478184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:56:46.094785921Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=info ts=2026-05-12T22:56:47.49568966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:56:47.500974179Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T22:56:47.941245659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:56:47.946851544Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:56:48.845431268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:56:48.848951294Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:56:49.38658976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:56:49.389326011Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-12T22:56:49.427247074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:56:49.4297101Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-12T22:56:49.429723223Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=4
ts=2026-05-12T22:56:50.288771292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=info ts=2026-05-12T22:56:50.338748388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T22:56:50.341756574Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=info ts=2026-05-12T22:56:50.819861846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:56:50.824399914Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:56:50.824421453Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:56:50.824444863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:56:50.826742558Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:56:51.473592423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:56:51.479534538Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-12T22:56:52.287281426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:56552"
level=debug ts=2026-05-12T22:56:52.761243835Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=warn ts=2026-05-12T22:56:54.427726965Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:56:54.517153668Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:56:54.521151961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T22:56:54.523993921Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T22:56:54.524011879Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:56:54.524030406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T22:56:54.52658468Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=debug ts=2026-05-12T22:56:56.094677142Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=info ts=2026-05-12T22:56:57.984292657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:56:57.988869622Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-12T22:56:58.223059407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:47128"
level=debug ts=2026-05-12T22:56:59.427977075Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-12T22:56:59.428009421Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:57:01.594415454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:57:01.597767075Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T22:57:02.761071609Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-12T22:57:03.517245656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9be421cde0f9c653 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="897 kB" avg_uncomp="897 kB" full=1 labels="{stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T22:57:03.520633683Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9be421cde0f9c653/19e1e285607%3A19e1e3051c9%3A2a80abf6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-felc\"}"
level=warn ts=2026-05-12T22:57:04.427682047Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:57:06.094938213Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=info ts=2026-05-12T22:57:06.883240008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:57:06.886501266Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-12T22:57:06.894626745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:57:06.89700072Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:57:09.349772732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T22:57:09.352510627Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=info ts=2026-05-12T22:57:09.427694159Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:57:09.427693794Z caller=checkpoint.go:384 msg="flushing series" totalSize="129 kB" series=5
level=debug ts=2026-05-12T22:57:09.427713823Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:57:09.427721629Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:57:09.427720967Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:57:09.43390995Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000016.tmp new=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-12T22:57:09.435161803Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007535356s
level=info ts=2026-05-12T22:57:09.540179283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:57:09.544875892Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-12T22:57:14.427480339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:57:15.91048246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:57:15.913216293Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:57:17.198671133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="516 B" avg_comp="516 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:57:17.20193191Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-12T22:57:19.427951135Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-12T22:57:20.290950351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T22:57:21.331091525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:48626"
level=info ts=2026-05-12T22:57:22.633405619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:57:22.637654007Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-12T22:57:24.42770407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:57:24.551084617Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:57:25.374916403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:57:25.37836451Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T22:57:26.764529312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:57:26.767177897Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T22:57:26.785225995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.5 MB" total_uncomp="160 MB" avg_uncomp="20 MB" full=7 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:57:26.791813078Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-12T22:57:28.225954724Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:35090"
level=debug ts=2026-05-12T22:57:28.618022333Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:57:28.619524571Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T22:57:29.427745001Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:57:31.364596141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:57:31.371463991Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:57:33.70065673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:57:33.704572303Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-12T22:57:34.427424105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:57:35.21602871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:57:35.22300488Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:57:36.744693275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:57:36.749570033Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:57:38.674146314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T22:57:38.67706421Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=info ts=2026-05-12T22:57:39.410814997Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:57:39.411094815Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:57:39.427319889Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:57:39.427338737Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:57:39.427346854Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T22:57:39.427357265Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-12T22:57:39.42740049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:57:39.427491105Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-12T22:57:39.427500702Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T22:57:39.427519364Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.471µs success=true
level=debug ts=2026-05-12T22:57:39.427764104Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=info ts=2026-05-12T22:57:42.688640554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:57:42.691472163Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-12T22:57:42.761705397Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=warn ts=2026-05-12T22:57:44.427678753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:57:45.650446276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T22:57:45.654757959Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=info ts=2026-05-12T22:57:45.715972685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:57:45.721137614Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T22:57:45.991781515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:57:45.994545469Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T22:57:46.094940616Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-12T22:57:48.382670909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:57:48.385997423Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:57:48.386017307Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:57:48.38603998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:57:48.388934749Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-12T22:57:49.428082447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:57:49.428090455Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-05-12T22:57:49.615965026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9be421cde0f9c653 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="897 kB" avg_uncomp="897 kB" full=1 labels="{stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T22:57:49.618938333Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9be421cde0f9c653/19e1e285607%3A19e1e3051c9%3A2a80abf6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-felc\"}"
ts=2026-05-12T22:57:50.294587639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:57:51.334077499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:58540"
ts=2026-05-12T22:57:52.291935337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:37580"
level=debug ts=2026-05-12T22:57:52.762083795Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-05-12T22:57:54.427774808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:57:55.935796908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:57:55.940742654Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-12T22:57:56.095592564Z caller=checkpoint.go:372 msg="writing series" size="586 kB"
level=info ts=2026-05-12T22:57:56.109403329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:57:56.11219937Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-12T22:57:57.633657957Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:57:58.642554242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:57:58.646853678Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-12T22:57:59.427761001Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:57:59.42801594Z caller=checkpoint.go:372 msg="writing series" size="536 kB"
level=debug ts=2026-05-12T22:57:59.428032125Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=7
level=debug ts=2026-05-12T22:58:02.761684792Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=warn ts=2026-05-12T22:58:04.427764173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:58:04.484879342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:58:04.488066053Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T22:58:06.094871071Z caller=checkpoint.go:372 msg="writing series" size="894 B"
level=info ts=2026-05-12T22:58:07.272412771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:58:07.275393586Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:58:07.927919478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:58:07.931085287Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T22:58:09.427415842Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:58:09.427436523Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:58:09.42746301Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:58:09.427470934Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T22:58:09.42863045Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-12T22:58:11.693592445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.5 MB" total_uncomp="160 MB" avg_uncomp="20 MB" full=7 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:58:11.700159444Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:58:12.762332391Z caller=checkpoint.go:372 msg="writing series" size="662 kB"
level=debug ts=2026-05-12T22:58:12.762358219Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=warn ts=2026-05-12T22:58:14.427982838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:58:16.094861097Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=info ts=2026-05-12T22:58:16.198086737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:58:16.200808423Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:58:17.992199254Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:58:18.831406865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:58:18.834858202Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-12T22:58:19.427421545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:58:19.428511628Z caller=checkpoint.go:372 msg="writing series" size="1.0 kB"
level=info ts=2026-05-12T22:58:19.613395398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:58:19.616138899Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-12T22:58:20.29776569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:58:21.337670502Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:35900"
ts=2026-05-12T22:58:22.293677325Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:51722"
level=debug ts=2026-05-12T22:58:22.762078669Z caller=checkpoint.go:372 msg="writing series" size="819 kB"
level=warn ts=2026-05-12T22:58:24.427691636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:58:25.002855923Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:58:25.002862573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=cf8cb9a5706fbae0 immediate=false num_chunks=1 total_comp="84 kB" avg_comp="84 kB" total_uncomp="412 kB" avg_uncomp="412 kB" idle=1 labels="{stream=\"chains-e2e-fqnw\"}"
level=error ts=2026-05-12T22:58:25.006607026Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/cf8cb9a5706fbae0/19e1e278ed0%3A19e1e325793%3Ac55b4380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-fqnw\"}"
level=info ts=2026-05-12T22:58:25.303276419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="325 B" avg_comp="325 B" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T22:58:25.306131744Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e176530%3A19e1e32a1cb%3A1bac6966?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-12T22:58:26.094688726Z caller=checkpoint.go:372 msg="writing series" size="6.2 kB"
level=info ts=2026-05-12T22:58:26.242277307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:58:26.248386255Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T22:58:27.034878613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T22:58:27.038111068Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=warn ts=2026-05-12T22:58:29.427484052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:58:29.490978582Z caller=checkpoint.go:372 msg="writing series" size="55 MB"
level=debug ts=2026-05-12T22:58:29.491013532Z caller=checkpoint.go:384 msg="flushing series" totalSize="56 MB" series=5
level=info ts=2026-05-12T22:58:32.172051566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:58:32.179841153Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T22:58:32.761679192Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-12T22:58:34.232356741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:58:34.236280976Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T22:58:34.238315459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:58:34.240599503Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-12T22:58:34.427567151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:58:34.893187216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:58:34.896038964Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T22:58:34.957933562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:58:34.960352728Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:58:36.033145093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="626 B" avg_comp="626 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T22:58:36.03616456Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=debug ts=2026-05-12T22:58:36.09534874Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=info ts=2026-05-12T22:58:36.256816803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=cf8cb9a5706fbae0 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-fqnw\"}"
level=error ts=2026-05-12T22:58:36.259766972Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/cf8cb9a5706fbae0/19e1e278ed0%3A19e1e325793%3Ac55b4380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-fqnw\"}"
level=info ts=2026-05-12T22:58:37.412893014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9be421cde0f9c653 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="897 kB" avg_uncomp="897 kB" full=1 labels="{stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T22:58:37.416078899Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9be421cde0f9c653/19e1e285607%3A19e1e3051c9%3A2a80abf6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-felc\"}"
level=debug ts=2026-05-12T22:58:39.410894854Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:58:39.410902036Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T22:58:39.427239482Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:58:39.427254798Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:58:39.427262424Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:58:39.427284599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:58:39.428499393Z caller=checkpoint.go:372 msg="writing series" size="225 kB"
level=debug ts=2026-05-12T22:58:42.763860938Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-12T22:58:42.763882418Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=4
level=info ts=2026-05-12T22:58:43.08287859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:58:43.088660557Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T22:58:44.077012408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T22:58:44.07987033Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e176530%3A19e1e32a1cb%3A1bac6966?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-12T22:58:44.427953421Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:58:46.098761114Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-12T22:58:46.098785052Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=1
level=info ts=2026-05-12T22:58:47.32868217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:58:47.332922394Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-12T22:58:49.427276042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:58:49.42841546Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=info ts=2026-05-12T22:58:50.282532171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:58:50.288143865Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:58:50.28816637Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:58:50.288188416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:58:50.293167672Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
ts=2026-05-12T22:58:50.300406367Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T22:58:51.340539712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:60990"
level=debug ts=2026-05-12T22:58:52.76122218Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-12T22:58:53.437679571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:58:53.440292087Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:58:54.044890512Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T22:58:54.427221254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:58:54.655241668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:58:54.657770479Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T22:58:54.831931907Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T22:58:55.194037529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.5 MB" total_uncomp="160 MB" avg_uncomp="20 MB" full=7 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:58:55.202344124Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T22:58:56.115382161Z caller=checkpoint.go:372 msg="writing series" size="22 MB"
level=debug ts=2026-05-12T22:58:56.115405874Z caller=checkpoint.go:384 msg="flushing series" totalSize="22 MB" series=3
level=info ts=2026-05-12T22:58:56.340003367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:58:56.343086749Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-05-12T22:58:58.233060404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:48182"
level=warn ts=2026-05-12T22:58:59.427597944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:58:59.428649325Z caller=checkpoint.go:372 msg="writing series" size="1.6 kB"
level=info ts=2026-05-12T22:59:02.633496352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:59:02.639819006Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T22:59:02.762383524Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-12T22:59:03.11023435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=cf8cb9a5706fbae0 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-fqnw\"}"
level=error ts=2026-05-12T22:59:03.113375311Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/cf8cb9a5706fbae0/19e1e278ed0%3A19e1e325793%3Ac55b4380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-fqnw\"}"
level=warn ts=2026-05-12T22:59:04.427440596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:59:04.75270569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:59:04.758110512Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-12T22:59:05.681905398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="684 B" avg_comp="684 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:59:05.685262723Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:59:06.099168033Z caller=checkpoint.go:372 msg="writing series" size="3.4 MB"
level=debug ts=2026-05-12T22:59:06.099195509Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.7 MB" series=3
level=info ts=2026-05-12T22:59:09.427325569Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:59:09.427356728Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:59:09.427365546Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:59:09.427398507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:59:09.428664079Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-12T22:59:12.761707535Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-05-12T22:59:14.427454142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:59:16.094797835Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=info ts=2026-05-12T22:59:16.937881303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T22:59:16.943274176Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T22:59:18.046518154Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T22:59:19.097420581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T22:59:19.101325948Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T22:59:19.254826764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T22:59:19.258412527Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e176530%3A19e1e32a1cb%3A1bac6966?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-12T22:59:19.427721953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:59:19.427775166Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
level=info ts=2026-05-12T22:59:19.493882602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T22:59:19.497443513Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-12T22:59:20.302467034Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=info ts=2026-05-12T22:59:21.967617422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T22:59:21.972486262Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=info ts=2026-05-12T22:59:21.983532134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T22:59:21.989729655Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T22:59:22.762115202Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-12T22:59:24.427716398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:59:25.395613975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9be421cde0f9c653 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="897 kB" avg_uncomp="897 kB" full=1 labels="{stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T22:59:25.3990866Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9be421cde0f9c653/19e1e285607%3A19e1e3051c9%3A2a80abf6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-felc\"}"
level=debug ts=2026-05-12T22:59:26.095347077Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-12T22:59:27.044091586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:59:27.050713126Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:59:27.050732989Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T22:59:27.050755236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:59:27.057292245Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:59:28.573420277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T22:59:28.577584424Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T22:59:29.427292394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:59:29.428389322Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-12T22:59:29.561657696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T22:59:29.564469423Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=info ts=2026-05-12T22:59:31.487976058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T22:59:31.491020176Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-12T22:59:31.57540513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T22:59:31.578073617Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-12T22:59:31.664412154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T22:59:31.671979211Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-12T22:59:32.762719423Z caller=checkpoint.go:372 msg="writing series" size="528 kB"
level=warn ts=2026-05-12T22:59:34.42769823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:59:36.094819555Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-12T22:59:36.520150001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T22:59:36.523242617Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T22:59:39.410833967Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T22:59:39.4111403Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T22:59:39.427404618Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T22:59:39.427422476Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T22:59:39.427429672Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T22:59:39.427452402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:59:39.428570106Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-05-12T22:59:39.428588437Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-05-12T22:59:40.79709359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T22:59:40.804358181Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T22:59:41.667568498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T22:59:41.670374242Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T22:59:42.762041396Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-12T22:59:43.672712276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="20 MB" avg_comp="2.4 MB" total_uncomp="160 MB" avg_uncomp="20 MB" full=7 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T22:59:43.68353677Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T22:59:44.311930475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T22:59:44.317209662Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-12T22:59:44.428000797Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:59:46.094795563Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-12T22:59:46.272418014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=cf8cb9a5706fbae0 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-fqnw\"}"
level=error ts=2026-05-12T22:59:46.27545949Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/cf8cb9a5706fbae0/19e1e278ed0%3A19e1e325793%3Ac55b4380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-fqnw\"}"
level=info ts=2026-05-12T22:59:46.865273294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T22:59:46.868858162Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-12T22:59:49.427342124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:59:49.428653003Z caller=checkpoint.go:372 msg="writing series" size="483 kB"
level=info ts=2026-05-12T22:59:49.592760976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T22:59:49.597016122Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-12T22:59:50.304233797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=debug ts=2026-05-12T22:59:51.929028737Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T22:59:51.948268993Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-12T22:59:52.300908221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:42632"
level=debug ts=2026-05-12T22:59:52.76212437Z caller=checkpoint.go:372 msg="writing series" size="899 kB"
level=debug ts=2026-05-12T22:59:52.76214598Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=4
level=warn ts=2026-05-12T22:59:54.427528553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T22:59:55.580340584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T22:59:55.583819252Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T22:59:56.095651622Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=warn ts=2026-05-12T22:59:59.428064721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T22:59:59.42807331Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-12T23:00:00.80199631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T23:00:00.805282489Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e176530%3A19e1e32a1cb%3A1bac6966?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-12T23:00:02.761127139Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=warn ts=2026-05-12T23:00:04.428119895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:00:06.094782669Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-05-12T23:00:08.812347948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T23:00:08.818837286Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T23:00:09.428050254Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T23:00:09.42806355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:00:09.428076418Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T23:00:09.42808456Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T23:00:09.429061595Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-12T23:00:09.429074751Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=5
level=debug ts=2026-05-12T23:00:12.761867973Z caller=checkpoint.go:372 msg="writing series" size="940 kB"
level=info ts=2026-05-12T23:00:12.790642472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T23:00:12.801586633Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T23:00:12.860927028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T23:00:12.866944372Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T23:00:12.961148178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T23:00:12.963852709Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T23:00:14.427527858Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:00:14.452620686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T23:00:14.460890465Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T23:00:16.095390104Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=debug ts=2026-05-12T23:00:16.095410395Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=info ts=2026-05-12T23:00:17.115404758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T23:00:17.118630147Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T23:00:17.731338748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T23:00:17.734419045Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=warn ts=2026-05-12T23:00:19.427205817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:00:19.428241216Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-12T23:00:19.872615905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T23:00:19.876396275Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-12T23:00:20.200980006Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-12T23:00:20.306106403Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=debug ts=2026-05-12T23:00:22.761498563Z caller=checkpoint.go:372 msg="writing series" size="9.4 kB"
level=info ts=2026-05-12T23:00:23.186662055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T23:00:23.189682236Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T23:00:24.186425114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9be421cde0f9c653 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="897 kB" avg_uncomp="897 kB" full=1 labels="{stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T23:00:24.194896581Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9be421cde0f9c653/19e1e285607%3A19e1e3051c9%3A2a80abf6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-felc\"}"
level=warn ts=2026-05-12T23:00:24.427469721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:00:24.998348458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T23:00:25.001382925Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T23:00:25.001413017Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:00:25.001435749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T23:00:25.003964424Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=info ts=2026-05-12T23:00:25.182724512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T23:00:25.188236693Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-12T23:00:25.740499971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=9 total_comp="22 MB" avg_comp="2.4 MB" total_uncomp="179 MB" avg_uncomp="20 MB" full=8 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T23:00:25.748464586Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T23:00:25.748488065Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:00:25.748514001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=9 total_comp="22 MB" avg_comp="2.4 MB" total_uncomp="179 MB" avg_uncomp="20 MB" full=8 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T23:00:25.754444676Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T23:00:26.095521117Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
level=info ts=2026-05-12T23:00:27.571594028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T23:00:27.575152443Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T23:00:27.841049993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T23:00:27.843797451Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-12T23:00:29.428130875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:00:29.428281921Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=info ts=2026-05-12T23:00:32.109886888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T23:00:32.112867214Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T23:00:32.761644376Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-12T23:00:34.428009277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:00:36.095500811Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=debug ts=2026-05-12T23:00:38.097391686Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T23:00:38.29992884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T23:00:38.302621364Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T23:00:38.698855298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=9 total_comp="22 MB" avg_comp="2.4 MB" total_uncomp="179 MB" avg_uncomp="20 MB" full=8 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T23:00:38.705708505Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T23:00:39.410855056Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T23:00:39.411029494Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=debug ts=2026-05-12T23:00:39.412292389Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-12T23:00:39.412359411Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976251 n=1
level=debug ts=2026-05-12T23:00:39.412402074Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976251 n=1
level=info ts=2026-05-12T23:00:39.427282612Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T23:00:39.427310093Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T23:00:39.427318251Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T23:00:39.427343091Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:00:39.428608Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=info ts=2026-05-12T23:00:41.258346062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=cf8cb9a5706fbae0 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-fqnw\"}"
level=error ts=2026-05-12T23:00:41.261415085Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/cf8cb9a5706fbae0/19e1e278ed0%3A19e1e325793%3Ac55b4380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-fqnw\"}"
level=info ts=2026-05-12T23:00:42.354647183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T23:00:42.357558418Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=debug ts=2026-05-12T23:00:42.761859184Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
level=info ts=2026-05-12T23:00:43.119769192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T23:00:43.123255029Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-12T23:00:43.123275587Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:00:43.123325052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T23:00:43.126061543Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-12T23:00:44.427951713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:00:45.630770484Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T23:00:46.094985804Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=debug ts=2026-05-12T23:00:46.095010618Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=warn ts=2026-05-12T23:00:49.427693126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:00:49.427815297Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=info ts=2026-05-12T23:00:49.603114093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T23:00:49.607198082Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-12T23:00:50.308165868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T23:00:51.349484372Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:34234"
level=debug ts=2026-05-12T23:00:52.762713538Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-12T23:00:52.762735369Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=2
level=info ts=2026-05-12T23:00:53.661347873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T23:00:53.664857347Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T23:00:54.427773187Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:00:55.315564733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T23:00:55.31890181Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e176530%3A19e1e32a1cb%3A1bac6966?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-12T23:00:56.096420474Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-12T23:00:56.096442901Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=1
level=info ts=2026-05-12T23:00:56.847441086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T23:00:56.853226107Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T23:00:57.722757023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T23:00:57.728321537Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T23:00:57.728342783Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:00:57.72836613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T23:00:57.733754546Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-12T23:00:59.427969228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:00:59.42811169Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=info ts=2026-05-12T23:01:00.592519007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T23:01:00.595981286Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T23:01:01.782407998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T23:01:01.785392835Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T23:01:02.761628028Z caller=checkpoint.go:372 msg="writing series" size="792 kB"
level=info ts=2026-05-12T23:01:03.775348917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T23:01:03.778226898Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=warn ts=2026-05-12T23:01:04.427897581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:01:05.222239011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T23:01:05.225206883Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T23:01:06.094623382Z caller=checkpoint.go:372 msg="writing series" size="570 kB"
level=debug ts=2026-05-12T23:01:06.09464689Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=info ts=2026-05-12T23:01:09.427990101Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T23:01:09.428031068Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-12T23:01:09.428024012Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:01:09.428041211Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T23:01:09.428275577Z caller=checkpoint.go:372 msg="writing series" size="272 kB"
level=info ts=2026-05-12T23:01:09.648092791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T23:01:09.652092459Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T23:01:10.136627919Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T23:01:10.705320239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=9 total_comp="22 MB" avg_comp="2.4 MB" total_uncomp="179 MB" avg_uncomp="20 MB" full=8 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T23:01:10.71277244Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T23:01:11.280179614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T23:01:11.286480687Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T23:01:11.629706801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9be421cde0f9c653 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="897 kB" avg_uncomp="897 kB" full=1 labels="{stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T23:01:11.632871887Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9be421cde0f9c653/19e1e285607%3A19e1e3051c9%3A2a80abf6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-felc\"}"
level=info ts=2026-05-12T23:01:11.925742543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T23:01:11.932992947Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T23:01:12.761621376Z caller=checkpoint.go:372 msg="writing series" size="267 kB"
level=warn ts=2026-05-12T23:01:14.428241197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:01:15.588952424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T23:01:15.59192311Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-12T23:01:16.095425158Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=info ts=2026-05-12T23:01:16.904720212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T23:01:16.907484081Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=info ts=2026-05-12T23:01:17.81684225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T23:01:17.819818962Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-12T23:01:19.427788787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:01:19.42786597Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
ts=2026-05-12T23:01:20.310070136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=info ts=2026-05-12T23:01:20.693777024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T23:01:20.696321451Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T23:01:21.329169643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T23:01:21.332399203Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T23:01:22.132160075Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T23:01:22.133782618Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-12T23:01:22.761806163Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=info ts=2026-05-12T23:01:23.394567235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T23:01:23.397527388Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T23:01:23.636511078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T23:01:23.641533262Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-12T23:01:24.427934542Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:01:26.094959916Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=info ts=2026-05-12T23:01:26.678835959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T23:01:26.682422733Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T23:01:26.682441795Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:01:26.682469167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T23:01:26.685049972Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-12T23:01:28.246773311Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:43828"
level=warn ts=2026-05-12T23:01:29.427344372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:01:29.429448138Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T23:01:29.429467525Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=7
level=info ts=2026-05-12T23:01:30.071169723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T23:01:30.075399129Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T23:01:31.991168862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=cf8cb9a5706fbae0 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-fqnw\"}"
level=error ts=2026-05-12T23:01:31.994447251Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/cf8cb9a5706fbae0/19e1e278ed0%3A19e1e325793%3Ac55b4380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-fqnw\"}"
level=debug ts=2026-05-12T23:01:32.762178688Z caller=checkpoint.go:372 msg="writing series" size="738 kB"
level=info ts=2026-05-12T23:01:33.989459597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T23:01:33.99252147Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T23:01:33.99254083Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:01:33.992562052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T23:01:33.995165749Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T23:01:34.428032465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:01:36.094861482Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=debug ts=2026-05-12T23:01:36.094883814Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=debug ts=2026-05-12T23:01:39.248448106Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T23:01:39.410969365Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T23:01:39.411356501Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T23:01:39.427687144Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T23:01:39.427699Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=debug ts=2026-05-12T23:01:39.427707293Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T23:01:39.427721064Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T23:01:39.427728213Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:01:40.182832914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="516 B" avg_comp="516 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T23:01:40.186124575Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T23:01:42.761936251Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=warn ts=2026-05-12T23:01:44.428026063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:01:45.753581197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T23:01:45.758038812Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T23:01:45.814282025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T23:01:45.817140658Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e176530%3A19e1e32a1cb%3A1bac6966?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-12T23:01:46.094904386Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=info ts=2026-05-12T23:01:46.232411081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T23:01:46.238026516Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T23:01:46.645279374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T23:01:46.648624587Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=info ts=2026-05-12T23:01:49.289174352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T23:01:49.294974682Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-12T23:01:49.427348364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:01:49.43031688Z caller=checkpoint.go:372 msg="writing series" size="2.8 MB"
level=debug ts=2026-05-12T23:01:49.430329654Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=4
level=info ts=2026-05-12T23:01:49.488494363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T23:01:49.491123974Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-12T23:01:50.312379764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T23:01:51.354013169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:53366"
level=debug ts=2026-05-12T23:01:52.761516033Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-12T23:01:54.427550638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:01:54.503807165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T23:01:54.51137888Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T23:01:54.661050343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T23:01:54.664368394Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T23:01:54.875605682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=9 total_comp="22 MB" avg_comp="2.4 MB" total_uncomp="179 MB" avg_uncomp="20 MB" full=8 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T23:01:54.882592362Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T23:01:56.094481213Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-12T23:01:57.736758064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T23:01:57.741011443Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
ts=2026-05-12T23:01:58.249599563Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:51226"
level=info ts=2026-05-12T23:01:58.807677244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T23:01:58.811087546Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-12T23:01:58.811106371Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:01:58.811127022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T23:01:58.813879374Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T23:01:59.427849138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:01:59.427884481Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=info ts=2026-05-12T23:01:59.62681489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9be421cde0f9c653 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="897 kB" avg_uncomp="897 kB" full=1 labels="{stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T23:01:59.629861596Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9be421cde0f9c653/19e1e285607%3A19e1e3051c9%3A2a80abf6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T23:01:59.629887111Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:01:59.629914442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9be421cde0f9c653 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="897 kB" avg_uncomp="897 kB" full=1 labels="{stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T23:01:59.632400911Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9be421cde0f9c653/19e1e285607%3A19e1e3051c9%3A2a80abf6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-felc\"}"
level=debug ts=2026-05-12T23:02:02.7616453Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-12T23:02:04.427415728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:02:06.076452187Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-12T23:02:06.094525035Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=info ts=2026-05-12T23:02:08.792156091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T23:02:08.795524807Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T23:02:09.428145094Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T23:02:09.42814775Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:02:09.428149658Z caller=checkpoint.go:384 msg="flushing series" totalSize="132 kB" series=5
level=debug ts=2026-05-12T23:02:09.428169664Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T23:02:09.428177288Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T23:02:09.432324346Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000017.tmp new=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-12T23:02:09.43354911Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.00590191s
level=info ts=2026-05-12T23:02:10.471028718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T23:02:10.474105081Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T23:02:12.04882894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T23:02:12.051795552Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-12T23:02:12.454989549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=cf8cb9a5706fbae0 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-fqnw\"}"
level=error ts=2026-05-12T23:02:12.457809804Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/cf8cb9a5706fbae0/19e1e278ed0%3A19e1e325793%3Ac55b4380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-fqnw\"}"
level=info ts=2026-05-12T23:02:12.720542397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T23:02:12.724160935Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T23:02:13.554639307Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T23:02:14.427550405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:02:15.862112508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T23:02:15.867178137Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-12T23:02:16.142660769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T23:02:16.145573148Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T23:02:16.145590579Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:02:16.145612302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T23:02:16.147739198Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-12T23:02:17.157577089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9be421cde0f9c653 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="897 kB" avg_uncomp="897 kB" full=1 labels="{stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T23:02:17.160682264Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9be421cde0f9c653/19e1e285607%3A19e1e3051c9%3A2a80abf6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-felc\"}"
level=info ts=2026-05-12T23:02:17.586937969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T23:02:17.589508456Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T23:02:19.428186987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:02:19.44727699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T23:02:19.449834294Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-12T23:02:19.449850652Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:02:19.449871379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T23:02:19.451910966Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T23:02:20.26813942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T23:02:20.270787462Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-12T23:02:20.315148633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T23:02:22.314446252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:52468"
level=info ts=2026-05-12T23:02:22.454640987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T23:02:22.457447756Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-12T23:02:22.45746675Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:02:22.457488324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T23:02:22.459624105Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-12T23:02:24.42812369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:02:28.260073941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T23:02:28.265582563Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-12T23:02:29.427339252Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:02:29.7400007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T23:02:29.743439954Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e176530%3A19e1e32a1cb%3A1bac6966?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-12T23:02:29.89135639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T23:02:29.896730694Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T23:02:30.73322241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T23:02:30.739200987Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T23:02:30.739221959Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:02:30.739246832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T23:02:30.744216019Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-12T23:02:34.427988477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:02:36.84012494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T23:02:36.848576001Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T23:02:37.464270198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T23:02:37.46698373Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=info ts=2026-05-12T23:02:37.972332141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T23:02:37.975213476Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T23:02:37.975238031Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:02:37.975261793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T23:02:37.977431797Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T23:02:38.033766448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T23:02:38.036114069Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T23:02:39.410844897Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T23:02:39.411536076Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T23:02:39.427805643Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-12T23:02:39.427837786Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T23:02:39.427839768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:02:39.427855992Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T23:02:39.427864282Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-12T23:02:39.427917375Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000018
level=info ts=2026-05-12T23:02:39.427925975Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T23:02:39.427945345Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.332µs success=true
level=debug ts=2026-05-12T23:02:39.428126342Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=info ts=2026-05-12T23:02:39.898679873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T23:02:39.901786785Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T23:02:40.039581455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T23:02:40.042207105Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T23:02:40.053340346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=9 total_comp="22 MB" avg_comp="2.4 MB" total_uncomp="179 MB" avg_uncomp="20 MB" full=8 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T23:02:40.059222099Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T23:02:41.198910772Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T23:02:42.762493485Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=warn ts=2026-05-12T23:02:44.42774455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:02:46.095189362Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-12T23:02:49.42720295Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:02:49.428322051Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-05-12T23:02:49.631076504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T23:02:49.636414429Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T23:02:50.158994996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T23:02:50.161762604Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-12T23:02:50.29101322Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T23:02:50.319056165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=info ts=2026-05-12T23:02:50.529469441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T23:02:50.532068454Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=info ts=2026-05-12T23:02:50.673488404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T23:02:50.676977934Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T23:02:51.545499701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9be421cde0f9c653 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="897 kB" avg_uncomp="897 kB" full=1 labels="{stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T23:02:51.548464756Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9be421cde0f9c653/19e1e285607%3A19e1e3051c9%3A2a80abf6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-felc\"}"
level=info ts=2026-05-12T23:02:52.483002153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T23:02:52.48576211Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-12T23:02:52.761533859Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-05-12T23:02:54.428053139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:02:56.096030984Z caller=checkpoint.go:372 msg="writing series" size="705 kB"
level=info ts=2026-05-12T23:02:59.426231716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=warn ts=2026-05-12T23:02:59.427895079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:02:59.428468981Z caller=checkpoint.go:372 msg="writing series" size="664 kB"
level=debug ts=2026-05-12T23:02:59.428488172Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=7
level=error ts=2026-05-12T23:02:59.431469841Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T23:03:00.588779347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T23:03:00.59160459Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T23:03:01.325580257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T23:03:01.328358628Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T23:03:02.040598235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T23:03:02.043536439Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T23:03:02.737525329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T23:03:02.740329304Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T23:03:02.762479513Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=info ts=2026-05-12T23:03:03.473852814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T23:03:03.479425665Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-12T23:03:04.427410691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:03:05.801974775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T23:03:05.804911356Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-12T23:03:05.804929795Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:03:05.804951636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T23:03:05.807170084Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T23:03:06.094836995Z caller=checkpoint.go:372 msg="writing series" size="894 B"
level=info ts=2026-05-12T23:03:06.626541783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=cf8cb9a5706fbae0 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-fqnw\"}"
level=error ts=2026-05-12T23:03:06.629394471Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/cf8cb9a5706fbae0/19e1e278ed0%3A19e1e325793%3Ac55b4380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-fqnw\"}"
level=info ts=2026-05-12T23:03:09.427261868Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T23:03:09.427277018Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:03:09.427288953Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T23:03:09.427321322Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T23:03:09.42842368Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-12T23:03:09.579835385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T23:03:09.583031753Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-05-12T23:03:09.583053197Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:03:09.583074794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T23:03:09.585252385Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-12T23:03:11.981850981Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T23:03:11.981895832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=775a79487da4dce7 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" idle=1 labels="{stream=\"gitlab-rep-zjeb\"}"
level=error ts=2026-05-12T23:03:11.985796029Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/775a79487da4dce7/19e1e28ddca%3A19e1e36fc1a%3A3eaa9d50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-zjeb\"}"
level=info ts=2026-05-12T23:03:12.282660609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=55884472be662265 immediate=false num_chunks=1 total_comp="538 kB" avg_comp="538 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" idle=1 labels="{stream=\"konflux-bflt\"}"
level=error ts=2026-05-12T23:03:12.293256213Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/55884472be662265/19e1e28977b%3A19e1e36fc97%3Aa5385589?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-bflt\"}"
level=debug ts=2026-05-12T23:03:12.762009437Z caller=checkpoint.go:372 msg="writing series" size="662 kB"
level=debug ts=2026-05-12T23:03:12.762032828Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-12T23:03:13.298345334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T23:03:13.303508374Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-12T23:03:14.428022063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:03:16.09536245Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=info ts=2026-05-12T23:03:17.836084359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T23:03:17.839322251Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-12T23:03:19.427869727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:03:19.428972057Z caller=checkpoint.go:372 msg="writing series" size="1.0 kB"
ts=2026-05-12T23:03:20.322656284Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=info ts=2026-05-12T23:03:22.275601911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=775a79487da4dce7 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-zjeb\"}"
level=error ts=2026-05-12T23:03:22.279504774Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/775a79487da4dce7/19e1e28ddca%3A19e1e36fc1a%3A3eaa9d50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-zjeb\"}"
ts=2026-05-12T23:03:22.31871864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:41686"
level=debug ts=2026-05-12T23:03:22.762013084Z caller=checkpoint.go:372 msg="writing series" size="686 kB"
level=info ts=2026-05-12T23:03:23.28208899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T23:03:23.285505076Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T23:03:23.454155249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=55884472be662265 immediate=false num_chunks=1 total_comp="605 kB" avg_comp="605 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-bflt\"}"
level=error ts=2026-05-12T23:03:23.457946385Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/55884472be662265/19e1e28977b%3A19e1e36fc97%3Aa5385589?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-bflt\"}"
level=warn ts=2026-05-12T23:03:24.427527276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:03:24.66238436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T23:03:24.665483243Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e176530%3A19e1e32a1cb%3A1bac6966?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-12T23:03:24.681501932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T23:03:24.684059157Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T23:03:26.094840458Z caller=checkpoint.go:372 msg="writing series" size="6.2 kB"
level=info ts=2026-05-12T23:03:27.343748544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T23:03:27.34659382Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-12T23:03:28.107793933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T23:03:28.113123586Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-12T23:03:28.25578158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:59180"
level=warn ts=2026-05-12T23:03:29.427361845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:03:29.48046397Z caller=checkpoint.go:372 msg="writing series" size="55 MB"
level=debug ts=2026-05-12T23:03:29.480490199Z caller=checkpoint.go:384 msg="flushing series" totalSize="56 MB" series=5
level=debug ts=2026-05-12T23:03:32.761877792Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-12T23:03:33.308442462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T23:03:33.314376026Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-12T23:03:34.427431901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:03:35.221501389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T23:03:35.228283811Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T23:03:36.095720178Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=info ts=2026-05-12T23:03:37.292237366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="626 B" avg_comp="626 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T23:03:37.294688108Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=info ts=2026-05-12T23:03:38.362561006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=9 total_comp="22 MB" avg_comp="2.4 MB" total_uncomp="179 MB" avg_uncomp="20 MB" full=8 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T23:03:38.370234491Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T23:03:39.4107379Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T23:03:39.411801791Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-12T23:03:39.428103726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:03:39.428114065Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T23:03:39.428128183Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T23:03:39.428136013Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T23:03:39.428256924Z caller=checkpoint.go:372 msg="writing series" size="225 kB"
level=debug ts=2026-05-12T23:03:40.550914054Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T23:03:41.99420712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T23:03:41.997555363Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T23:03:42.76410244Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-12T23:03:42.764123876Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=4
level=debug ts=2026-05-12T23:03:43.236923698Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T23:03:43.703153712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T23:03:43.706194158Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T23:03:44.381890911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T23:03:44.384615739Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T23:03:44.384635652Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:03:44.384656248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T23:03:44.386912894Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=warn ts=2026-05-12T23:03:44.427678163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:03:44.499846687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T23:03:44.50348523Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T23:03:46.098008014Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-12T23:03:46.098035018Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=1
level=info ts=2026-05-12T23:03:46.513246513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T23:03:46.516046744Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T23:03:49.266375772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="684 B" avg_comp="684 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T23:03:49.269971316Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-12T23:03:49.427283168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:03:49.428427033Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
ts=2026-05-12T23:03:50.325262573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=info ts=2026-05-12T23:03:51.476913787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9be421cde0f9c653 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="897 kB" avg_uncomp="897 kB" full=1 labels="{stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T23:03:51.480091954Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9be421cde0f9c653/19e1e285607%3A19e1e3051c9%3A2a80abf6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-felc\"}"
level=debug ts=2026-05-12T23:03:52.762539875Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-12T23:03:54.427224462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:03:54.58679147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T23:03:54.592474198Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-12T23:03:55.933480181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=55884472be662265 immediate=false num_chunks=1 total_comp="605 kB" avg_comp="605 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-bflt\"}"
level=error ts=2026-05-12T23:03:55.937767209Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/55884472be662265/19e1e28977b%3A19e1e36fc97%3Aa5385589?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-bflt\"}"
level=debug ts=2026-05-12T23:03:56.11904445Z caller=checkpoint.go:372 msg="writing series" size="23 MB"
level=debug ts=2026-05-12T23:03:56.119070374Z caller=checkpoint.go:384 msg="flushing series" totalSize="24 MB" series=3
ts=2026-05-12T23:03:58.258763522Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:51224"
level=info ts=2026-05-12T23:03:59.280562083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T23:03:59.283858045Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T23:03:59.318033329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T23:03:59.320555746Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T23:03:59.427903163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:03:59.428980325Z caller=checkpoint.go:372 msg="writing series" size="1.6 kB"
level=info ts=2026-05-12T23:04:01.081355991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T23:04:01.084438782Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=info ts=2026-05-12T23:04:01.631244808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=775a79487da4dce7 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-zjeb\"}"
level=error ts=2026-05-12T23:04:01.63521696Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/775a79487da4dce7/19e1e28ddca%3A19e1e36fc1a%3A3eaa9d50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-zjeb\"}"
level=info ts=2026-05-12T23:04:01.849704649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T23:04:01.852681549Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-12T23:04:02.762714416Z caller=checkpoint.go:372 msg="writing series" size="302 kB"
level=info ts=2026-05-12T23:04:03.366104371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T23:04:03.373716337Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T23:04:04.146137078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=cf8cb9a5706fbae0 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-fqnw\"}"
level=error ts=2026-05-12T23:04:04.149454691Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/cf8cb9a5706fbae0/19e1e278ed0%3A19e1e325793%3Ac55b4380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-fqnw\"}"
level=warn ts=2026-05-12T23:04:04.427911155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:04:06.101013263Z caller=checkpoint.go:372 msg="writing series" size="3.5 MB"
level=debug ts=2026-05-12T23:04:06.10103735Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=3
level=info ts=2026-05-12T23:04:06.558350947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T23:04:06.561498065Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T23:04:09.427543987Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T23:04:09.427574761Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T23:04:09.427583307Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T23:04:09.427578768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:04:09.428874804Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-12T23:04:12.761960261Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-12T23:04:13.175039955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T23:04:13.178769651Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T23:04:13.178792558Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:04:13.178814363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T23:04:13.18118629Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-12T23:04:14.428147761Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:04:15.059503782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T23:04:15.06607304Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T23:04:16.095996803Z caller=checkpoint.go:372 msg="writing series" size="325 kB"
level=info ts=2026-05-12T23:04:16.178971414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T23:04:16.18178634Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e176530%3A19e1e32a1cb%3A1bac6966?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-12T23:04:17.498712077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T23:04:17.501536394Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-12T23:04:18.136444402Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T23:04:19.427632925Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:04:19.428753507Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
level=info ts=2026-05-12T23:04:19.647836241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T23:04:19.651197112Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
ts=2026-05-12T23:04:20.32745803Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T23:04:21.364270149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:33112"
level=debug ts=2026-05-12T23:04:22.762435795Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=warn ts=2026-05-12T23:04:24.428061733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:04:26.094899624Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-12T23:04:28.374618288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T23:04:28.382416046Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T23:04:28.49285791Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T23:04:29.427405467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:04:29.428534056Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-12T23:04:29.550742189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T23:04:29.553599228Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=info ts=2026-05-12T23:04:31.768566958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T23:04:31.771621486Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T23:04:32.087372675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T23:04:32.090181407Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T23:04:32.762574001Z caller=checkpoint.go:372 msg="writing series" size="401 kB"
level=info ts=2026-05-12T23:04:32.996722687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T23:04:32.999416293Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T23:04:33.878249724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T23:04:33.884645031Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-12T23:04:34.427247176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:04:34.756632367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T23:04:34.760236808Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T23:04:35.43933112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T23:04:35.442443258Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T23:04:36.095564219Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-12T23:04:37.090435431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=9 total_comp="22 MB" avg_comp="2.4 MB" total_uncomp="179 MB" avg_uncomp="20 MB" full=8 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T23:04:37.09834546Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T23:04:39.410874404Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T23:04:39.410889599Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T23:04:39.42717867Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T23:04:39.427198982Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T23:04:39.427207014Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T23:04:39.427218618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:04:39.428364577Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-12T23:04:40.845610419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9be421cde0f9c653 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="897 kB" avg_uncomp="897 kB" full=1 labels="{stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T23:04:40.848750263Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9be421cde0f9c653/19e1e285607%3A19e1e3051c9%3A2a80abf6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-felc\"}"
level=info ts=2026-05-12T23:04:41.260816601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T23:04:41.263603859Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T23:04:42.762019111Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=debug ts=2026-05-12T23:04:42.762042276Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-05-12T23:04:44.42802106Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:04:46.095735321Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=debug ts=2026-05-12T23:04:48.528624589Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T23:04:49.427784797Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:04:49.429045626Z caller=checkpoint.go:372 msg="writing series" size="490 kB"
ts=2026-05-12T23:04:50.330353773Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=info ts=2026-05-12T23:04:50.540281211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=55884472be662265 immediate=false num_chunks=1 total_comp="605 kB" avg_comp="605 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-bflt\"}"
level=error ts=2026-05-12T23:04:50.54696453Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/55884472be662265/19e1e28977b%3A19e1e36fc97%3Aa5385589?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-bflt\"}"
level=info ts=2026-05-12T23:04:50.707507077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T23:04:50.71049192Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-12T23:04:51.929342145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=cf8cb9a5706fbae0 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-fqnw\"}"
level=error ts=2026-05-12T23:04:51.935106094Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/cf8cb9a5706fbae0/19e1e278ed0%3A19e1e325793%3Ac55b4380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-fqnw\"}"
level=error ts=2026-05-12T23:04:51.935128003Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:04:51.935151143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=cf8cb9a5706fbae0 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-fqnw\"}"
level=error ts=2026-05-12T23:04:51.937917028Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/cf8cb9a5706fbae0/19e1e278ed0%3A19e1e325793%3Ac55b4380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-fqnw\"}"
level=info ts=2026-05-12T23:04:52.047331721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T23:04:52.052549773Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
ts=2026-05-12T23:04:52.326684764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:32822"
level=debug ts=2026-05-12T23:04:52.762674638Z caller=checkpoint.go:372 msg="writing series" size="899 kB"
level=debug ts=2026-05-12T23:04:52.762696013Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=info ts=2026-05-12T23:04:53.576436308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=775a79487da4dce7 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-zjeb\"}"
level=error ts=2026-05-12T23:04:53.580043693Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/775a79487da4dce7/19e1e28ddca%3A19e1e36fc1a%3A3eaa9d50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-zjeb\"}"
level=warn ts=2026-05-12T23:04:54.427975712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:04:55.2982407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T23:04:55.301281584Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T23:04:56.09562749Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=info ts=2026-05-12T23:04:58.464347566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T23:04:58.467403686Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T23:04:58.650879465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T23:04:58.655796625Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T23:04:58.943535712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T23:04:58.946339042Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T23:04:59.427430276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:04:59.428504497Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-12T23:05:00.458166386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T23:05:00.461092472Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=debug ts=2026-05-12T23:05:02.761663712Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=warn ts=2026-05-12T23:05:04.427348485Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:05:04.874924241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=cf8cb9a5706fbae0 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-fqnw\"}"
level=error ts=2026-05-12T23:05:04.877749013Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/cf8cb9a5706fbae0/19e1e278ed0%3A19e1e325793%3Ac55b4380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-fqnw\"}"
level=info ts=2026-05-12T23:05:05.990081722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T23:05:05.992836263Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-12T23:05:06.095351523Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-05-12T23:05:09.427732522Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T23:05:09.42775946Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T23:05:09.427767027Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T23:05:09.427815792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:05:09.430047559Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T23:05:09.430059699Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=5
level=info ts=2026-05-12T23:05:09.518164102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T23:05:09.52555362Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T23:05:11.785879822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T23:05:11.788715863Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e176530%3A19e1e32a1cb%3A1bac6966?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-12T23:05:11.788736969Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:05:11.788758084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T23:05:11.791512291Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e176530%3A19e1e32a1cb%3A1bac6966?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-12T23:05:12.762904951Z caller=checkpoint.go:372 msg="writing series" size="940 kB"
level=warn ts=2026-05-12T23:05:14.427724267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:05:16.095877732Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=debug ts=2026-05-12T23:05:16.095895126Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=info ts=2026-05-12T23:05:17.316591384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T23:05:17.319557295Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T23:05:17.838317105Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T23:05:17.839917978Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T23:05:19.428043721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:05:19.429155422Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
ts=2026-05-12T23:05:20.333815165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T23:05:21.370580448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:35742"
level=info ts=2026-05-12T23:05:21.416586405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9be421cde0f9c653 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="897 kB" avg_uncomp="897 kB" full=1 labels="{stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T23:05:21.419753057Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9be421cde0f9c653/19e1e285607%3A19e1e3051c9%3A2a80abf6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-felc\"}"
level=debug ts=2026-05-12T23:05:22.343726518Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T23:05:22.343777221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="457 kB" avg_comp="457 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T23:05:22.347641011Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e1695e0%3A19e1e6cda9a%3Ad92154c8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T23:05:22.761818351Z caller=checkpoint.go:372 msg="writing series" size="9.4 kB"
level=info ts=2026-05-12T23:05:22.820208744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=9 total_comp="22 MB" avg_comp="2.4 MB" total_uncomp="179 MB" avg_uncomp="20 MB" full=8 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T23:05:22.82765148Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T23:05:23.97369997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T23:05:23.976666049Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e176530%3A19e1e32a1cb%3A1bac6966?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-12T23:05:24.428116521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:05:25.421898278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T23:05:25.428794178Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T23:05:25.81266323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T23:05:25.816537648Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T23:05:26.09523916Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
level=info ts=2026-05-12T23:05:26.299794517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T23:05:26.302829928Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=info ts=2026-05-12T23:05:28.190029158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T23:05:28.193021266Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T23:05:28.751731677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T23:05:28.755748141Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T23:05:29.427315321Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:05:29.428631176Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=info ts=2026-05-12T23:05:30.17477386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T23:05:30.177369288Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T23:05:32.564497313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T23:05:32.567834656Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T23:05:32.762449751Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-05-12T23:05:33.911741639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=55884472be662265 immediate=false num_chunks=1 total_comp="605 kB" avg_comp="605 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-bflt\"}"
level=error ts=2026-05-12T23:05:33.915882257Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/55884472be662265/19e1e28977b%3A19e1e36fc97%3Aa5385589?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-bflt\"}"
level=warn ts=2026-05-12T23:05:34.4274395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:05:36.095774699Z caller=checkpoint.go:372 msg="writing series" size="238 kB"
level=info ts=2026-05-12T23:05:37.436355836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="458 kB" avg_comp="458 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T23:05:37.441590877Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e1695e0%3A19e1e6cda9a%3Ad92154c8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T23:05:39.410611213Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T23:05:39.411240001Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T23:05:39.427619813Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T23:05:39.427645412Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T23:05:39.427653097Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T23:05:39.427645746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:05:39.429024154Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=info ts=2026-05-12T23:05:39.869130618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T23:05:39.8722214Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-12T23:05:40.036486621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=cf8cb9a5706fbae0 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-fqnw\"}"
level=error ts=2026-05-12T23:05:40.03936822Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/cf8cb9a5706fbae0/19e1e278ed0%3A19e1e325793%3Ac55b4380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-fqnw\"}"
level=info ts=2026-05-12T23:05:41.923275067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=775a79487da4dce7 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-zjeb\"}"
level=error ts=2026-05-12T23:05:41.926501942Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/775a79487da4dce7/19e1e28ddca%3A19e1e36fc1a%3A3eaa9d50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-zjeb\"}"
level=debug ts=2026-05-12T23:05:42.761948712Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=debug ts=2026-05-12T23:05:42.761972259Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-05-12T23:05:43.24935632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T23:05:43.254924492Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-12T23:05:44.428155502Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:05:46.09596502Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=debug ts=2026-05-12T23:05:46.784923755Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T23:05:49.427983543Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:05:49.429141581Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
ts=2026-05-12T23:05:50.335977532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=info ts=2026-05-12T23:05:50.48693288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T23:05:50.490133334Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T23:05:50.594339307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T23:05:50.597117375Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-12T23:05:51.275131616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T23:05:51.282274146Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-05-12T23:05:51.282293013Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:05:51.282334209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T23:05:51.287151143Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
ts=2026-05-12T23:05:51.373832825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:37876"
level=info ts=2026-05-12T23:05:51.403643073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T23:05:51.406445772Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T23:05:52.764123147Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-12T23:05:52.764149949Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=3
level=info ts=2026-05-12T23:05:53.204709527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T23:05:53.207691759Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=info ts=2026-05-12T23:05:53.845370133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T23:05:53.848398518Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T23:05:54.42760228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:05:54.541936857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T23:05:54.544604129Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e176530%3A19e1e32a1cb%3A1bac6966?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-12T23:05:56.097703562Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-12T23:05:56.097723727Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=1
level=debug ts=2026-05-12T23:05:56.948360837Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T23:05:57.77452052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="458 kB" avg_comp="458 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T23:05:57.778079239Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e1695e0%3A19e1e6cda9a%3Ad92154c8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-12T23:05:59.428103132Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:05:59.428276926Z caller=checkpoint.go:372 msg="writing series" size="177 kB"
level=debug ts=2026-05-12T23:06:02.762734867Z caller=checkpoint.go:372 msg="writing series" size="606 kB"
level=info ts=2026-05-12T23:06:04.274878329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T23:06:04.28085805Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-12T23:06:04.428174139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:06:06.095980337Z caller=checkpoint.go:372 msg="writing series" size="563 kB"
level=debug ts=2026-05-12T23:06:06.096009772Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=info ts=2026-05-12T23:06:06.763315775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T23:06:06.771450292Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T23:06:06.771477452Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:06:06.771498563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T23:06:06.777293841Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T23:06:07.108674923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T23:06:07.112048062Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T23:06:08.637342539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T23:06:08.641347308Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T23:06:09.053404278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9be421cde0f9c653 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="897 kB" avg_uncomp="897 kB" full=1 labels="{stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T23:06:09.056428029Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9be421cde0f9c653/19e1e285607%3A19e1e3051c9%3A2a80abf6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-felc\"}"
level=info ts=2026-05-12T23:06:09.427978649Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T23:06:09.428002421Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T23:06:09.428010204Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T23:06:09.428043618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:06:09.429414056Z caller=checkpoint.go:372 msg="writing series" size="302 kB"
level=info ts=2026-05-12T23:06:09.761565751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T23:06:09.764243564Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T23:06:09.943340149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T23:06:09.945965914Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=info ts=2026-05-12T23:06:10.846053264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T23:06:10.850845755Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-12T23:06:11.979205394Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T23:06:12.762586615Z caller=checkpoint.go:372 msg="writing series" size="267 kB"
level=info ts=2026-05-12T23:06:13.397160935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=9 total_comp="22 MB" avg_comp="2.4 MB" total_uncomp="179 MB" avg_uncomp="20 MB" full=8 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T23:06:13.403581349Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T23:06:14.427862518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:06:16.095768499Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=info ts=2026-05-12T23:06:18.076366856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T23:06:18.079106056Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T23:06:19.427591928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:06:19.428705485Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
ts=2026-05-12T23:06:20.338691238Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T23:06:22.762525477Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
level=info ts=2026-05-12T23:06:23.182644503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T23:06:23.189871888Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T23:06:24.427499822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:06:24.698362139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T23:06:24.701149292Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T23:06:26.094905429Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=debug ts=2026-05-12T23:06:26.09492912Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-12T23:06:26.896317262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T23:06:26.89956924Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-12T23:06:28.272904006Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:50950"
level=warn ts=2026-05-12T23:06:29.428030066Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:06:29.429809802Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T23:06:29.429824841Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
level=info ts=2026-05-12T23:06:29.514293248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=55884472be662265 immediate=false num_chunks=1 total_comp="605 kB" avg_comp="605 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-bflt\"}"
level=error ts=2026-05-12T23:06:29.518048003Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/55884472be662265/19e1e28977b%3A19e1e36fc97%3Aa5385589?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-bflt\"}"
level=info ts=2026-05-12T23:06:30.149479076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=cf8cb9a5706fbae0 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-fqnw\"}"
level=error ts=2026-05-12T23:06:30.153088905Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/cf8cb9a5706fbae0/19e1e278ed0%3A19e1e325793%3Ac55b4380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-fqnw\"}"
level=info ts=2026-05-12T23:06:30.158338062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T23:06:30.161076397Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-12T23:06:31.657672344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T23:06:31.660618787Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T23:06:32.761920164Z caller=checkpoint.go:372 msg="writing series" size="611 kB"
level=warn ts=2026-05-12T23:06:34.427573517Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:06:36.095472341Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=debug ts=2026-05-12T23:06:36.095496176Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=info ts=2026-05-12T23:06:36.097343975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T23:06:36.103083964Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-12T23:06:38.868169801Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T23:06:39.410757578Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T23:06:39.411156878Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T23:06:39.427621449Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T23:06:39.427639469Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T23:06:39.427646793Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T23:06:39.427644962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:06:39.428748818Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-12T23:06:39.997177298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T23:06:40.005127198Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T23:06:40.018329312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=775a79487da4dce7 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-zjeb\"}"
level=error ts=2026-05-12T23:06:40.021209161Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/775a79487da4dce7/19e1e28ddca%3A19e1e36fc1a%3A3eaa9d50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-zjeb\"}"
level=info ts=2026-05-12T23:06:41.515874731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T23:06:41.519199442Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-12T23:06:42.070771934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="458 kB" avg_comp="458 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T23:06:42.074610846Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e1695e0%3A19e1e6cda9a%3Ad92154c8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T23:06:42.558952244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T23:06:42.561820997Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e176530%3A19e1e32a1cb%3A1bac6966?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-12T23:06:42.762604163Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=info ts=2026-05-12T23:06:43.866098637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T23:06:43.86926979Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T23:06:43.869289743Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:06:43.869328649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T23:06:43.871540949Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=warn ts=2026-05-12T23:06:44.427413604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:06:46.030904767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T23:06:46.035681259Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-12T23:06:46.095830238Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-12T23:06:46.299328526Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T23:06:49.427235329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:06:49.43075096Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-12T23:06:49.430765335Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=4
ts=2026-05-12T23:06:50.340889033Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T23:06:51.37924656Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:51618"
level=info ts=2026-05-12T23:06:52.054633619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T23:06:52.057670069Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-05-12T23:06:52.336921893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:57514"
level=info ts=2026-05-12T23:06:52.572126615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T23:06:52.574811133Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T23:06:52.762433842Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-05-12T23:06:53.61204855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T23:06:53.61924586Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-12T23:06:54.427574954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:06:55.141505303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T23:06:55.144232956Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=debug ts=2026-05-12T23:06:56.095341328Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-12T23:06:57.388203776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T23:06:57.392363471Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-12T23:06:57.392385515Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:06:57.392410581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T23:06:57.395413446Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-05-12T23:06:58.275922159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:57520"
level=warn ts=2026-05-12T23:06:59.427496101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:06:59.428634396Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=info ts=2026-05-12T23:06:59.815029483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T23:06:59.81805364Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T23:07:00.768328317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9be421cde0f9c653 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="897 kB" avg_uncomp="897 kB" full=1 labels="{stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T23:07:00.771436837Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9be421cde0f9c653/19e1e285607%3A19e1e3051c9%3A2a80abf6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-felc\"}"
level=info ts=2026-05-12T23:07:02.458273065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T23:07:02.465109833Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T23:07:02.761911535Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-12T23:07:03.070863962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T23:07:03.073740801Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T23:07:04.427531443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:07:06.095176393Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=info ts=2026-05-12T23:07:07.346039434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=9 total_comp="22 MB" avg_comp="2.4 MB" total_uncomp="179 MB" avg_uncomp="20 MB" full=8 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T23:07:07.353906761Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T23:07:07.353929065Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:07:07.353957172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=9 total_comp="22 MB" avg_comp="2.4 MB" total_uncomp="179 MB" avg_uncomp="20 MB" full=8 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T23:07:07.361121182Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T23:07:07.88365572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T23:07:07.886576175Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=info ts=2026-05-12T23:07:09.114024448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T23:07:09.117440595Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T23:07:09.427381994Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T23:07:09.427414096Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T23:07:09.427426271Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T23:07:09.427442557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:07:09.428485434Z caller=checkpoint.go:384 msg="flushing series" totalSize="133 kB" series=5
level=info ts=2026-05-12T23:07:09.436473789Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000018.tmp new=/var/loki/wal/checkpoint.000018
level=info ts=2026-05-12T23:07:09.437749877Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009675366s
level=debug ts=2026-05-12T23:07:10.770420818Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T23:07:12.189012422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T23:07:12.192694998Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T23:07:14.0815063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=55884472be662265 immediate=false num_chunks=1 total_comp="605 kB" avg_comp="605 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-bflt\"}"
level=error ts=2026-05-12T23:07:14.085714847Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/55884472be662265/19e1e28977b%3A19e1e36fc97%3Aa5385589?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-bflt\"}"
level=info ts=2026-05-12T23:07:14.141919671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T23:07:14.145733726Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T23:07:14.42767327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:07:16.448158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="516 B" avg_comp="516 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T23:07:16.451046894Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-12T23:07:19.42757705Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:07:19.577929529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T23:07:19.581002446Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
ts=2026-05-12T23:07:20.34243504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=info ts=2026-05-12T23:07:21.732592555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=9 total_comp="22 MB" avg_comp="2.4 MB" total_uncomp="179 MB" avg_uncomp="20 MB" full=8 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T23:07:21.739149841Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T23:07:22.14998897Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T23:07:22.289350478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="458 kB" avg_comp="458 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T23:07:22.292795217Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e1695e0%3A19e1e6cda9a%3Ad92154c8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T23:07:22.972765551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=cf8cb9a5706fbae0 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-fqnw\"}"
level=error ts=2026-05-12T23:07:22.975700944Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/cf8cb9a5706fbae0/19e1e278ed0%3A19e1e325793%3Ac55b4380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-fqnw\"}"
level=warn ts=2026-05-12T23:07:24.427747054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:07:26.135655792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T23:07:26.140708484Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-12T23:07:26.913279066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T23:07:26.91604748Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-12T23:07:28.279214098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:44390"
level=info ts=2026-05-12T23:07:28.480017984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=775a79487da4dce7 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-zjeb\"}"
level=error ts=2026-05-12T23:07:28.483483181Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/775a79487da4dce7/19e1e28ddca%3A19e1e36fc1a%3A3eaa9d50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-zjeb\"}"
level=warn ts=2026-05-12T23:07:29.427817849Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:07:33.75635282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T23:07:33.759400725Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T23:07:34.428257954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:07:35.092904585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T23:07:35.098417602Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T23:07:35.09843791Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:07:35.098461228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T23:07:35.103359389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T23:07:35.103574358Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T23:07:35.107904153Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T23:07:35.771618085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T23:07:35.774368875Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e176530%3A19e1e32a1cb%3A1bac6966?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-12T23:07:37.49810045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T23:07:37.501373633Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T23:07:38.52321344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T23:07:38.526112161Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T23:07:39.410758869Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T23:07:39.410812984Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-12T23:07:39.427158096Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-12T23:07:39.427175383Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T23:07:39.427192014Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T23:07:39.427202505Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T23:07:39.427212505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:07:39.427314308Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000019
level=info ts=2026-05-12T23:07:39.427322855Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T23:07:39.427341165Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.615µs success=true
level=debug ts=2026-05-12T23:07:39.427542899Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=info ts=2026-05-12T23:07:39.852121328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T23:07:39.8550953Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-12T23:07:42.761803587Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=debug ts=2026-05-12T23:07:43.776850909Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T23:07:44.427572913Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:07:45.183753807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T23:07:45.192447628Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-12T23:07:45.47971016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T23:07:45.485550929Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T23:07:46.095207991Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-12T23:07:46.470407082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9be421cde0f9c653 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="897 kB" avg_uncomp="897 kB" full=1 labels="{stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T23:07:46.47355288Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9be421cde0f9c653/19e1e285607%3A19e1e3051c9%3A2a80abf6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-felc\"}"
level=info ts=2026-05-12T23:07:47.913499175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T23:07:47.916931137Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=warn ts=2026-05-12T23:07:49.42797552Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:07:49.428003153Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-05-12T23:07:50.127104738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T23:07:50.130170767Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-12T23:07:50.344366951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=info ts=2026-05-12T23:07:50.986798764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=9 total_comp="22 MB" avg_comp="2.4 MB" total_uncomp="179 MB" avg_uncomp="20 MB" full=8 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T23:07:50.994015317Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T23:07:52.761824478Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-05-12T23:07:54.427795381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:07:55.145792712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T23:07:55.148380121Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T23:07:55.443410167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T23:07:55.449793355Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T23:07:56.09621411Z caller=checkpoint.go:372 msg="writing series" size="576 kB"
level=debug ts=2026-05-12T23:07:57.357201653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T23:07:59.427292405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:07:59.429089073Z caller=checkpoint.go:372 msg="writing series" size="597 kB"
level=debug ts=2026-05-12T23:07:59.429103695Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=debug ts=2026-05-12T23:08:02.761668386Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=info ts=2026-05-12T23:08:03.23288114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=cf8cb9a5706fbae0 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-fqnw\"}"
level=error ts=2026-05-12T23:08:03.235825731Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/cf8cb9a5706fbae0/19e1e278ed0%3A19e1e325793%3Ac55b4380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-fqnw\"}"
level=warn ts=2026-05-12T23:08:04.42794474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:08:06.094911949Z caller=checkpoint.go:372 msg="writing series" size="894 B"
level=info ts=2026-05-12T23:08:07.270432054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="458 kB" avg_comp="458 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T23:08:07.274349891Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e1695e0%3A19e1e6cda9a%3Ad92154c8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T23:08:07.731747822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T23:08:07.734809742Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T23:08:08.349030607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=55884472be662265 immediate=false num_chunks=1 total_comp="605 kB" avg_comp="605 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-bflt\"}"
level=error ts=2026-05-12T23:08:08.352761747Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/55884472be662265/19e1e28977b%3A19e1e36fc97%3Aa5385589?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-bflt\"}"
level=info ts=2026-05-12T23:08:09.253413559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T23:08:09.256971405Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T23:08:09.427791849Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T23:08:09.427816187Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T23:08:09.427823858Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T23:08:09.427841988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:08:09.427986701Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-12T23:08:10.144910089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T23:08:10.147686819Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-12T23:08:12.094343092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T23:08:12.097290344Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T23:08:12.097327895Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:08:12.097350588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T23:08:12.099668276Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T23:08:12.288343265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T23:08:12.291013831Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-12T23:08:12.761749935Z caller=checkpoint.go:372 msg="writing series" size="662 kB"
level=debug ts=2026-05-12T23:08:12.761775988Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-12T23:08:13.396150123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=775a79487da4dce7 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-zjeb\"}"
level=error ts=2026-05-12T23:08:13.399689541Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/775a79487da4dce7/19e1e28ddca%3A19e1e36fc1a%3A3eaa9d50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-zjeb\"}"
level=warn ts=2026-05-12T23:08:14.427354012Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:08:16.094765899Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=debug ts=2026-05-12T23:08:17.08252759Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T23:08:19.000739179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="626 B" avg_comp="626 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T23:08:19.004456899Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=info ts=2026-05-12T23:08:19.110787033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T23:08:19.116868629Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-12T23:08:19.427215217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:08:19.428347241Z caller=checkpoint.go:372 msg="writing series" size="1.0 kB"
level=info ts=2026-05-12T23:08:20.116224119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T23:08:20.119032078Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T23:08:20.119052406Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:08:20.119073601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T23:08:20.121332693Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-12T23:08:20.346990971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T23:08:22.343614806Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:44478"
level=debug ts=2026-05-12T23:08:22.762103719Z caller=checkpoint.go:372 msg="writing series" size="748 kB"
level=warn ts=2026-05-12T23:08:24.427681332Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:08:26.094478184Z caller=checkpoint.go:372 msg="writing series" size="6.2 kB"
level=info ts=2026-05-12T23:08:27.264811878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T23:08:27.26954235Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T23:08:27.269562937Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:08:27.269590501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T23:08:27.271923849Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T23:08:28.015065192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T23:08:28.017827131Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-12T23:08:29.428098551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:08:29.485398953Z caller=checkpoint.go:372 msg="writing series" size="56 MB"
level=debug ts=2026-05-12T23:08:29.485428112Z caller=checkpoint.go:384 msg="flushing series" totalSize="56 MB" series=5
level=info ts=2026-05-12T23:08:30.462206428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T23:08:30.464984Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=info ts=2026-05-12T23:08:31.161785111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T23:08:31.16462872Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-12T23:08:32.152590668Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T23:08:32.15428327Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-12T23:08:32.205490667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T23:08:32.208294319Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e176530%3A19e1e32a1cb%3A1bac6966?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-12T23:08:32.21134519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T23:08:32.213692418Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-12T23:08:32.213707053Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:08:32.213726053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T23:08:32.215944687Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T23:08:32.761546495Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-12T23:08:34.331821142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9be421cde0f9c653 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="897 kB" avg_uncomp="897 kB" full=1 labels="{stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T23:08:34.335191485Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9be421cde0f9c653/19e1e285607%3A19e1e3051c9%3A2a80abf6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T23:08:34.335210335Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:08:34.335229374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9be421cde0f9c653 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="897 kB" avg_uncomp="897 kB" full=1 labels="{stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T23:08:34.338037228Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9be421cde0f9c653/19e1e285607%3A19e1e3051c9%3A2a80abf6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-felc\"}"
level=warn ts=2026-05-12T23:08:34.427944343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:08:34.482071072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T23:08:34.487292509Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T23:08:35.438111786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=9 total_comp="22 MB" avg_comp="2.4 MB" total_uncomp="179 MB" avg_uncomp="20 MB" full=8 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T23:08:35.446223962Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-12T23:08:36.095060431Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=info ts=2026-05-12T23:08:37.250502726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T23:08:37.254707778Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T23:08:39.410552659Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T23:08:39.411601763Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-12T23:08:39.42790677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:08:39.427912066Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T23:08:39.427933247Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T23:08:39.427944616Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T23:08:39.428102773Z caller=checkpoint.go:372 msg="writing series" size="225 kB"
level=info ts=2026-05-12T23:08:39.696670889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T23:08:39.699714745Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T23:08:39.756466996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="684 B" avg_comp="684 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T23:08:39.759238105Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-12T23:08:39.759255955Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:08:39.759274258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T23:08:39.761510775Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T23:08:42.762875916Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-12T23:08:42.762897352Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=4
level=info ts=2026-05-12T23:08:42.768441293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T23:08:42.771366374Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T23:08:42.795413548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T23:08:42.800690482Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-12T23:08:44.427241799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:08:45.915232689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=cf8cb9a5706fbae0 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-fqnw\"}"
level=error ts=2026-05-12T23:08:45.918335966Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/cf8cb9a5706fbae0/19e1e278ed0%3A19e1e325793%3Ac55b4380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-fqnw\"}"
level=debug ts=2026-05-12T23:08:46.09658921Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-12T23:08:46.096613341Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=1
level=info ts=2026-05-12T23:08:47.505491262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T23:08:47.509008515Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T23:08:49.428703161Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:08:49.428715951Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=debug ts=2026-05-12T23:08:50.22745051Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-12T23:08:50.350352931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=info ts=2026-05-12T23:08:50.702621958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T23:08:50.706354662Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-12T23:08:50.706375333Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:08:50.706396287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T23:08:50.709538633Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-12T23:08:51.393126814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:40994"
level=info ts=2026-05-12T23:08:52.116793391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="458 kB" avg_comp="458 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T23:08:52.120332929Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e1695e0%3A19e1e6cda9a%3Ad92154c8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T23:08:52.637611924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9be421cde0f9c653 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="897 kB" avg_uncomp="897 kB" full=1 labels="{stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T23:08:52.640518394Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9be421cde0f9c653/19e1e285607%3A19e1e3051c9%3A2a80abf6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-felc\"}"
level=debug ts=2026-05-12T23:08:52.761730994Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-12T23:08:53.085707908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T23:08:53.088470044Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-12T23:08:53.751439147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T23:08:53.759131309Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T23:08:54.427867492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:08:54.885402806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T23:08:54.888179781Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T23:08:56.069281775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=55884472be662265 immediate=false num_chunks=1 total_comp="605 kB" avg_comp="605 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-bflt\"}"
level=error ts=2026-05-12T23:08:56.07334068Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/55884472be662265/19e1e28977b%3A19e1e36fc97%3Aa5385589?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-bflt\"}"
level=debug ts=2026-05-12T23:08:56.117737471Z caller=checkpoint.go:372 msg="writing series" size="24 MB"
level=debug ts=2026-05-12T23:08:56.117759026Z caller=checkpoint.go:384 msg="flushing series" totalSize="24 MB" series=3
level=debug ts=2026-05-12T23:08:59.427756884Z caller=checkpoint.go:372 msg="writing series" size="1.6 kB"
level=warn ts=2026-05-12T23:08:59.427785628Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:08:59.618961964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T23:08:59.62232755Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T23:09:01.119965004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.8 kB" avg_uncomp="3.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T23:09:01.122791171Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e1e1766a8%3A19e1e1799f4%3Af660e1e6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-12T23:09:01.122813833Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:09:01.122835767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="182 B" avg_comp="182 B" total_uncomp="22 kB" avg_uncomp="22 kB" idle=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-12T23:09:01.1251725Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e1e176530%3A19e1e36fc0d%3A324e58f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-05-12T23:09:02.753442929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=775a79487da4dce7 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-zjeb\"}"
level=error ts=2026-05-12T23:09:02.756899758Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/775a79487da4dce7/19e1e28ddca%3A19e1e36fc1a%3A3eaa9d50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-zjeb\"}"
level=debug ts=2026-05-12T23:09:02.761289801Z caller=checkpoint.go:372 msg="writing series" size="359 kB"
level=warn ts=2026-05-12T23:09:04.428186228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:09:05.302627047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T23:09:05.306396046Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T23:09:06.065329176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T23:09:06.068359433Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T23:09:06.097388065Z caller=checkpoint.go:372 msg="writing series" size="3.3 MB"
level=debug ts=2026-05-12T23:09:06.097402202Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.7 MB" series=3
level=info ts=2026-05-12T23:09:06.581734652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T23:09:06.588983196Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-12T23:09:09.412673686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T23:09:09.415418477Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-12T23:09:09.427490503Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T23:09:09.427510045Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-12T23:09:09.427514396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:09:09.427521019Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T23:09:09.427771216Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-12T23:09:10.673515787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T23:09:10.676531355Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T23:09:12.761224471Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-12T23:09:13.904908626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-12T23:09:13.907872946Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e1e176530%3A19e1e36fc0d%3A324e58f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-05-12T23:09:14.427510467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:09:16.094786116Z caller=checkpoint.go:372 msg="writing series" size="412 kB"
level=debug ts=2026-05-12T23:09:17.093361383Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T23:09:17.518695303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T23:09:17.524280108Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T23:09:17.524325742Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:09:17.524349204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T23:09:17.529423488Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T23:09:17.785242154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T23:09:17.788631834Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T23:09:18.549242339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T23:09:18.553628887Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=info ts=2026-05-12T23:09:19.296746058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T23:09:19.300572547Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=warn ts=2026-05-12T23:09:19.427931065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:09:19.427971499Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
ts=2026-05-12T23:09:20.353282058Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=debug ts=2026-05-12T23:09:20.520068609Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T23:09:21.133496151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T23:09:21.136676021Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T23:09:22.470568364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T23:09:22.474191166Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e176530%3A19e1e32a1cb%3A1bac6966?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-12T23:09:22.761992506Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=info ts=2026-05-12T23:09:24.001028635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9be421cde0f9c653 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="897 kB" avg_uncomp="897 kB" full=1 labels="{stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T23:09:24.004795755Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9be421cde0f9c653/19e1e285607%3A19e1e3051c9%3A2a80abf6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-felc\"}"
level=warn ts=2026-05-12T23:09:24.427415766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:09:26.095208556Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-12T23:09:27.08497002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T23:09:27.092535876Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-12T23:09:29.427218125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:09:29.4283189Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-12T23:09:32.762246751Z caller=checkpoint.go:372 msg="writing series" size="401 kB"
level=debug ts=2026-05-12T23:09:32.762268328Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-05-12T23:09:32.976406645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T23:09:32.98201287Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T23:09:34.100012252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T23:09:34.103188317Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T23:09:34.219589029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=9 total_comp="22 MB" avg_comp="2.4 MB" total_uncomp="179 MB" avg_uncomp="20 MB" full=8 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T23:09:34.226748993Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T23:09:34.428271643Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:09:35.681829942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T23:09:35.684903012Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-12T23:09:35.684921346Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:09:35.684945936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T23:09:35.687484982Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T23:09:36.09463589Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-12T23:09:37.883693926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="458 kB" avg_comp="458 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T23:09:37.887743681Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e1695e0%3A19e1e6cda9a%3Ad92154c8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T23:09:39.410824107Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T23:09:39.410839158Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T23:09:39.427403732Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T23:09:39.427419134Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-12T23:09:39.427416038Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:09:39.427427022Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T23:09:39.427544966Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-12T23:09:40.389963201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=cf8cb9a5706fbae0 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-fqnw\"}"
level=error ts=2026-05-12T23:09:40.392884002Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/cf8cb9a5706fbae0/19e1e278ed0%3A19e1e325793%3Ac55b4380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-fqnw\"}"
level=info ts=2026-05-12T23:09:40.410387082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-12T23:09:40.412822626Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e1e176530%3A19e1e176531%3A577f475e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-05-12T23:09:40.412835923Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:09:40.412857089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="606 kB" avg_comp="606 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T23:09:40.416424311Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e169a68%3A19e1e6ce0e9%3Ace8c9c84?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T23:09:40.556730416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T23:09:40.561170818Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T23:09:40.716716427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=55884472be662265 immediate=false num_chunks=1 total_comp="605 kB" avg_comp="605 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-bflt\"}"
level=error ts=2026-05-12T23:09:40.720453928Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/55884472be662265/19e1e28977b%3A19e1e36fc97%3Aa5385589?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-bflt\"}"
level=error ts=2026-05-12T23:09:40.720476802Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:09:40.72049647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=55884472be662265 immediate=false num_chunks=1 total_comp="605 kB" avg_comp="605 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-bflt\"}"
level=error ts=2026-05-12T23:09:40.723810342Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/55884472be662265/19e1e28977b%3A19e1e36fc97%3Aa5385589?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-bflt\"}"
level=debug ts=2026-05-12T23:09:42.761666587Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-12T23:09:44.427510466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:09:44.999550045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T23:09:45.005623381Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-12T23:09:46.094586041Z caller=checkpoint.go:372 msg="writing series" size="5.6 kB"
level=info ts=2026-05-12T23:09:46.535056099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T23:09:46.537983881Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T23:09:49.215575844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-12T23:09:49.218620302Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e1e176530%3A19e1e36fc0d%3A324e58f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-05-12T23:09:49.428117699Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:09:49.42838809Z caller=checkpoint.go:372 msg="writing series" size="463 kB"
level=debug ts=2026-05-12T23:09:49.694990235Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-12T23:09:50.356645765Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=info ts=2026-05-12T23:09:51.076281988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T23:09:51.079367508Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T23:09:51.09039166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T23:09:51.09282814Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-12T23:09:52.761928551Z caller=checkpoint.go:372 msg="writing series" size="899 kB"
level=debug ts=2026-05-12T23:09:52.761951195Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=info ts=2026-05-12T23:09:53.622209546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="606 kB" avg_comp="606 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T23:09:53.626101941Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e169a68%3A19e1e6ce0e9%3Ace8c9c84?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-12T23:09:54.427259985Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:09:54.469457981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=55884472be662265 immediate=false num_chunks=1 total_comp="605 kB" avg_comp="605 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-bflt\"}"
level=error ts=2026-05-12T23:09:54.473479078Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/55884472be662265/19e1e28977b%3A19e1e36fc97%3Aa5385589?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-bflt\"}"
level=debug ts=2026-05-12T23:09:56.094546202Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=info ts=2026-05-12T23:09:57.985940431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T23:09:57.9892521Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-12T23:09:58.609219774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T23:09:58.615040247Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-12T23:09:59.427906237Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:09:59.427908937Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-12T23:10:00.471280494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=775a79487da4dce7 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-zjeb\"}"
level=error ts=2026-05-12T23:10:00.474805404Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/775a79487da4dce7/19e1e28ddca%3A19e1e36fc1a%3A3eaa9d50?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-zjeb\"}"
level=error ts=2026-05-12T23:10:00.474822938Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:10:00.474844845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7b03620a738d20e7 immediate=false num_chunks=1 total_comp="2.3 MB" avg_comp="2.3 MB" total_uncomp="15 MB" avg_uncomp="15 MB" idle=1 labels="{stream=\"build-e2e-lfwg\"}"
level=error ts=2026-05-12T23:10:00.499639372Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7b03620a738d20e7/19e1e28a71b%3A19e1e38b959%3A44ea025b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lfwg\"}"
level=info ts=2026-05-12T23:10:01.312042079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T23:10:01.317761291Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T23:10:02.738058237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T23:10:02.740878572Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T23:10:02.760999614Z caller=checkpoint.go:372 msg="writing series" size="6.2 kB"
level=warn ts=2026-05-12T23:10:04.427903809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:10:05.156472299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T23:10:05.159396864Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-12T23:10:06.028563858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T23:10:06.031422654Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=debug ts=2026-05-12T23:10:06.09454866Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-05-12T23:10:09.427216722Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T23:10:09.427233584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:10:09.427245425Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T23:10:09.42725401Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T23:10:09.429316678Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T23:10:09.429333455Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=5
level=info ts=2026-05-12T23:10:10.109362894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T23:10:10.11502173Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-12T23:10:10.91917034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T23:10:10.92290934Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-12T23:10:12.761631541Z caller=checkpoint.go:372 msg="writing series" size="960 kB"
level=warn ts=2026-05-12T23:10:14.427432178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:10:15.089094378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=9 total_comp="22 MB" avg_comp="2.4 MB" total_uncomp="179 MB" avg_uncomp="20 MB" full=8 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T23:10:15.096879713Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T23:10:15.226598108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T23:10:15.229473798Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-12T23:10:15.482650802Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T23:10:16.095146373Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=debug ts=2026-05-12T23:10:16.095167893Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=info ts=2026-05-12T23:10:16.30461506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T23:10:16.307802919Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e176530%3A19e1e32a1cb%3A1bac6966?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-12T23:10:17.375946446Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T23:10:18.36134905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1fca8197ff46ef5d immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T23:10:18.364754579Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1fca8197ff46ef5d/19e1e292fd2%3A19e1e2b1cf9%3Acd88461b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qhkh\"}"
level=error ts=2026-05-12T23:10:18.364776235Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:10:18.364800335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64bf9311b7fc1e3d immediate=false num_chunks=1 total_comp="1.9 kB" avg_comp="1.9 kB" total_uncomp="188 kB" avg_uncomp="188 kB" idle=1 labels="{stream=\"build-e2e-eolb\"}"
level=error ts=2026-05-12T23:10:18.367720387Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64bf9311b7fc1e3d/19e1e285dd8%3A19e1e3d2629%3Ad287ef44?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-eolb\"}"
level=info ts=2026-05-12T23:10:18.782919594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9be421cde0f9c653 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="897 kB" avg_uncomp="897 kB" full=1 labels="{stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T23:10:18.78668172Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9be421cde0f9c653/19e1e285607%3A19e1e3051c9%3A2a80abf6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-felc\"}"
level=warn ts=2026-05-12T23:10:19.427486778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:10:19.42751788Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-12T23:10:20.030350186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7b03620a738d20e7 immediate=false num_chunks=1 total_comp="598 kB" avg_comp="598 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-lfwg\"}"
level=error ts=2026-05-12T23:10:20.034925157Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7b03620a738d20e7/19e1e28a71b%3A19e1e38b959%3A44ea025b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lfwg\"}"
ts=2026-05-12T23:10:20.359881919Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
ts=2026-05-12T23:10:22.353920552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:42280"
level=debug ts=2026-05-12T23:10:22.761458754Z caller=checkpoint.go:372 msg="writing series" size="9.4 kB"
level=info ts=2026-05-12T23:10:23.877778543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T23:10:23.880712995Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T23:10:24.427269879Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:10:26.095137724Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
level=info ts=2026-05-12T23:10:26.974095475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T23:10:26.97756769Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T23:10:26.977589376Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:10:26.977610706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T23:10:26.980438932Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T23:10:28.439718359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=55884472be662265 immediate=false num_chunks=1 total_comp="605 kB" avg_comp="605 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-bflt\"}"
level=error ts=2026-05-12T23:10:28.443914448Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/55884472be662265/19e1e28977b%3A19e1e36fc97%3Aa5385589?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-bflt\"}"
level=warn ts=2026-05-12T23:10:29.427344184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:10:29.428774731Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-05-12T23:10:31.986533302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="606 kB" avg_comp="606 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T23:10:31.992028607Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e169a68%3A19e1e6ce0e9%3Ace8c9c84?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T23:10:32.761276454Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-05-12T23:10:32.953723387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T23:10:32.960332667Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T23:10:33.17591975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T23:10:33.17878867Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T23:10:33.907034957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=cf8cb9a5706fbae0 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-fqnw\"}"
level=error ts=2026-05-12T23:10:33.909944745Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/cf8cb9a5706fbae0/19e1e278ed0%3A19e1e325793%3Ac55b4380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-fqnw\"}"
level=warn ts=2026-05-12T23:10:34.427669105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:10:34.436806456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="458 kB" avg_comp="458 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T23:10:34.44081662Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e1695e0%3A19e1e6cda9a%3Ad92154c8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-12T23:10:35.663814966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64bf9311b7fc1e3d immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-eolb\"}"
level=error ts=2026-05-12T23:10:35.666894567Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64bf9311b7fc1e3d/19e1e285dd8%3A19e1e3d2629%3Ad287ef44?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-eolb\"}"
level=debug ts=2026-05-12T23:10:36.095228603Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=info ts=2026-05-12T23:10:39.41081379Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T23:10:39.410976578Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T23:10:39.427231138Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T23:10:39.427246307Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T23:10:39.427256439Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T23:10:39.427282315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:10:39.428564495Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=info ts=2026-05-12T23:10:39.743475431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T23:10:39.748729219Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T23:10:41.016874816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T23:10:41.01981248Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T23:10:41.43711213Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T23:10:42.299966469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-12T23:10:42.302777649Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e1e176530%3A19e1e36fc0d%3A324e58f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-05-12T23:10:42.761359764Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
level=debug ts=2026-05-12T23:10:42.761380101Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-12T23:10:44.427889522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:10:46.094847102Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
level=info ts=2026-05-12T23:10:48.772518406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T23:10:48.778199933Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-12T23:10:49.427827505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:10:49.427849416Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=info ts=2026-05-12T23:10:49.434818307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T23:10:49.437418538Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-12T23:10:50.361735236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
level=info ts=2026-05-12T23:10:50.616478833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7b03620a738d20e7 immediate=false num_chunks=1 total_comp="598 kB" avg_comp="598 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-lfwg\"}"
level=error ts=2026-05-12T23:10:50.62562408Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7b03620a738d20e7/19e1e28a71b%3A19e1e38b959%3A44ea025b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lfwg\"}"
ts=2026-05-12T23:10:51.40256513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:53786"
level=info ts=2026-05-12T23:10:52.701674103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T23:10:52.704609046Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T23:10:52.762665136Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-12T23:10:52.762694112Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=3
level=info ts=2026-05-12T23:10:53.193740765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T23:10:53.200208127Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-12T23:10:54.427726811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:10:56.096571333Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-12T23:10:56.096594049Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=1
ts=2026-05-12T23:10:58.294587768Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:39598"
level=warn ts=2026-05-12T23:10:59.427765672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:10:59.427924448Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=info ts=2026-05-12T23:10:59.9650288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T23:10:59.968289133Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T23:11:00.900832068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T23:11:00.904253962Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T23:11:01.641407364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T23:11:01.64473153Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T23:11:02.517200534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T23:11:02.519774983Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=debug ts=2026-05-12T23:11:02.76200092Z caller=checkpoint.go:372 msg="writing series" size="606 kB"
level=info ts=2026-05-12T23:11:02.843792981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T23:11:02.846605773Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-12T23:11:04.287669708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=9 total_comp="22 MB" avg_comp="2.4 MB" total_uncomp="179 MB" avg_uncomp="20 MB" full=8 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T23:11:04.294768694Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-12T23:11:04.427670718Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:11:04.871865897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T23:11:04.874285252Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e176530%3A19e1e32a1cb%3A1bac6966?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-12T23:11:06.095770435Z caller=checkpoint.go:372 msg="writing series" size="580 kB"
level=debug ts=2026-05-12T23:11:06.095788132Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=debug ts=2026-05-12T23:11:08.511002359Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-12T23:11:08.671408905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T23:11:08.67697794Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-12T23:11:09.428110991Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T23:11:09.428138542Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T23:11:09.428149509Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T23:11:09.428152466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:11:09.428535419Z caller=checkpoint.go:372 msg="writing series" size="333 kB"
level=info ts=2026-05-12T23:11:09.509157956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T23:11:09.511650101Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-12T23:11:09.799181487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9be421cde0f9c653 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="897 kB" avg_uncomp="897 kB" full=1 labels="{stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T23:11:09.802164774Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9be421cde0f9c653/19e1e285607%3A19e1e3051c9%3A2a80abf6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-felc\"}"
level=debug ts=2026-05-12T23:11:11.161478616Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T23:11:11.672042902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64bf9311b7fc1e3d immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-eolb\"}"
level=error ts=2026-05-12T23:11:11.675162766Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64bf9311b7fc1e3d/19e1e285dd8%3A19e1e3d2629%3Ad287ef44?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-eolb\"}"
level=debug ts=2026-05-12T23:11:12.761397867Z caller=checkpoint.go:372 msg="writing series" size="267 kB"
level=info ts=2026-05-12T23:11:12.997967979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="606 kB" avg_comp="606 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T23:11:13.003345417Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e169a68%3A19e1e6ce0e9%3Ace8c9c84?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T23:11:13.780444693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T23:11:13.78876936Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T23:11:14.42834199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:11:16.094670498Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=info ts=2026-05-12T23:11:19.122616161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="458 kB" avg_comp="458 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T23:11:19.126155822Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e1695e0%3A19e1e6cda9a%3Ad92154c8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T23:11:19.126177528Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:11:19.126201617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="458 kB" avg_comp="458 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T23:11:19.129200243Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e1695e0%3A19e1e6cda9a%3Ad92154c8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-12T23:11:19.428219466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:11:19.428267531Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
ts=2026-05-12T23:11:20.363887594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-nmsb9-8cd7643b 10.130.0.59:7946"
level=info ts=2026-05-12T23:11:21.099347027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T23:11:21.102641417Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-12T23:11:21.405357035Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:41222"
level=debug ts=2026-05-12T23:11:22.761709129Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=warn ts=2026-05-12T23:11:24.427891709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:11:25.365508568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T23:11:25.369546124Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-12T23:11:26.095005064Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=debug ts=2026-05-12T23:11:26.095027502Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-12T23:11:27.648821159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=55884472be662265 immediate=false num_chunks=1 total_comp="605 kB" avg_comp="605 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-bflt\"}"
level=error ts=2026-05-12T23:11:27.652468286Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/55884472be662265/19e1e28977b%3A19e1e36fc97%3Aa5385589?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-bflt\"}"
level=info ts=2026-05-12T23:11:29.38122045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="458 kB" avg_comp="458 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T23:11:29.384519709Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e1695e0%3A19e1e6cda9a%3Ad92154c8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-12T23:11:29.42721033Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:11:29.429140944Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-12T23:11:29.429153876Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
level=info ts=2026-05-12T23:11:30.70524126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=cf8cb9a5706fbae0 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-fqnw\"}"
level=error ts=2026-05-12T23:11:30.708424657Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/cf8cb9a5706fbae0/19e1e278ed0%3A19e1e325793%3Ac55b4380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-fqnw\"}"
level=error ts=2026-05-12T23:11:30.708449099Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:11:30.708471911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=cf8cb9a5706fbae0 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-fqnw\"}"
level=error ts=2026-05-12T23:11:30.710871283Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/cf8cb9a5706fbae0/19e1e278ed0%3A19e1e325793%3Ac55b4380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-fqnw\"}"
level=debug ts=2026-05-12T23:11:32.76208905Z caller=checkpoint.go:372 msg="writing series" size="619 kB"
level=info ts=2026-05-12T23:11:33.83670374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T23:11:33.8395836Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-12T23:11:34.428208575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:11:35.236804445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T23:11:35.242739923Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-12T23:11:36.094723252Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=debug ts=2026-05-12T23:11:36.094746134Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=info ts=2026-05-12T23:11:39.410852017Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T23:11:39.411630519Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T23:11:39.427904732Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T23:11:39.427923764Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T23:11:39.427931557Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T23:11:39.427938431Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=warn ts=2026-05-12T23:11:39.427950955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:11:42.01432534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-12T23:11:42.017624351Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e1e176530%3A19e1e36fc0d%3A324e58f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-05-12T23:11:42.761954997Z caller=checkpoint.go:372 msg="writing series" size="212 kB"
level=info ts=2026-05-12T23:11:42.936363069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T23:11:42.94034614Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-12T23:11:42.9624523Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-12T23:11:44.427735757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:11:44.779956728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T23:11:44.783264232Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=info ts=2026-05-12T23:11:45.925646227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T23:11:45.928386556Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T23:11:46.09488428Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-12T23:11:46.16705626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=cf8cb9a5706fbae0 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-fqnw\"}"
level=error ts=2026-05-12T23:11:46.169974708Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/cf8cb9a5706fbae0/19e1e278ed0%3A19e1e325793%3Ac55b4380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-fqnw\"}"
level=info ts=2026-05-12T23:11:46.910336242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T23:11:46.912701059Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T23:11:47.02610189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T23:11:47.030763474Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T23:11:47.20202336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=10 total_comp="24 MB" avg_comp="2.4 MB" total_uncomp="198 MB" avg_uncomp="20 MB" full=8 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T23:11:47.208643581Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T23:11:48.207347341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T23:11:48.210141313Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e176530%3A19e1e32a1cb%3A1bac6966?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-12T23:11:48.210160544Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:11:48.210181364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T23:11:48.212425844Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e176530%3A19e1e32a1cb%3A1bac6966?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-12T23:11:49.064619189Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-12T23:11:49.427481003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:11:49.429126666Z caller=checkpoint.go:372 msg="writing series" size="2.8 MB"
level=debug ts=2026-05-12T23:11:49.429142284Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=4
level=info ts=2026-05-12T23:11:50.25880799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7b03620a738d20e7 immediate=false num_chunks=1 total_comp="598 kB" avg_comp="598 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-lfwg\"}"
level=error ts=2026-05-12T23:11:50.262808841Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7b03620a738d20e7/19e1e28a71b%3A19e1e38b959%3A44ea025b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lfwg\"}"
ts=2026-05-12T23:11:50.365876735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-flqrb-992281bb 10.128.0.74:7946"
ts=2026-05-12T23:11:51.408136627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:60294"
level=info ts=2026-05-12T23:11:51.668357631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T23:11:51.672687564Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-12T23:11:52.761938345Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-05-12T23:11:53.934590104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="606 kB" avg_comp="606 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T23:11:53.93837016Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e169a68%3A19e1e6ce0e9%3Ace8c9c84?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-12T23:11:54.012681733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64bf9311b7fc1e3d immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-eolb\"}"
level=error ts=2026-05-12T23:11:54.015512717Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64bf9311b7fc1e3d/19e1e285dd8%3A19e1e3d2629%3Ad287ef44?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-eolb\"}"
level=warn ts=2026-05-12T23:11:54.427205815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:11:56.094780028Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-12T23:11:59.42777586Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:11:59.427818019Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=info ts=2026-05-12T23:12:01.138702589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9be421cde0f9c653 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="897 kB" avg_uncomp="897 kB" full=1 labels="{stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T23:12:01.141876843Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9be421cde0f9c653/19e1e285607%3A19e1e3051c9%3A2a80abf6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-felc\"}"
level=info ts=2026-05-12T23:12:01.368559229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T23:12:01.372131864Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-12T23:12:02.662887199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T23:12:02.665759858Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-12T23:12:02.760941727Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-12T23:12:04.427729268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:12:05.32661614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="458 kB" avg_comp="458 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T23:12:05.330133398Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e1695e0%3A19e1e6cda9a%3Ad92154c8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T23:12:06.094375459Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=info ts=2026-05-12T23:12:06.124605554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T23:12:06.127072249Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-12T23:12:06.287495666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T23:12:06.290251227Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e176530%3A19e1e32a1cb%3A1bac6966?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-12T23:12:07.254854815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T23:12:07.260382608Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-12T23:12:07.435107581Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T23:12:08.886211001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=cf8cb9a5706fbae0 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-fqnw\"}"
level=error ts=2026-05-12T23:12:08.889049255Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/cf8cb9a5706fbae0/19e1e278ed0%3A19e1e325793%3Ac55b4380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-fqnw\"}"
level=info ts=2026-05-12T23:12:09.427273968Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-12T23:12:09.427281226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:12:09.427324117Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T23:12:09.427339885Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-12T23:12:09.42834096Z caller=checkpoint.go:384 msg="flushing series" totalSize="134 kB" series=5
level=info ts=2026-05-12T23:12:09.435673908Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000019.tmp new=/var/loki/wal/checkpoint.000019
level=info ts=2026-05-12T23:12:09.436969042Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009485991s
level=info ts=2026-05-12T23:12:09.856762786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T23:12:09.862832121Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-12T23:12:14.427687491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:12:18.812414153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="516 B" avg_comp="516 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T23:12:18.816903251Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-12T23:12:19.427219406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:12:19.562571862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=55884472be662265 immediate=false num_chunks=1 total_comp="605 kB" avg_comp="605 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-bflt\"}"
level=error ts=2026-05-12T23:12:19.566558531Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/55884472be662265/19e1e28977b%3A19e1e36fc97%3Aa5385589?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-bflt\"}"
ts=2026-05-12T23:12:20.367728093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
level=info ts=2026-05-12T23:12:23.931233167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T23:12:23.936039229Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-12T23:12:24.427670442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:12:28.685245094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-12T23:12:28.690562782Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e1e176530%3A19e1e4d8ca5%3A90b8b24e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-12T23:12:29.427323544Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:12:30.065903772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T23:12:30.069093571Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=info ts=2026-05-12T23:12:31.503970913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T23:12:31.509205636Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T23:12:32.763485658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T23:12:32.766239407Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e176530%3A19e1e32a1cb%3A1bac6966?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-12T23:12:34.428150699Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:12:35.065000343Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-12T23:12:35.883087257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="525 kB" avg_uncomp="525 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-12T23:12:35.886951113Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e1e16a1d8%3A19e1e1968c3%3Ad4d938e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-12T23:12:38.955597626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="702 B" avg_comp="702 B" total_uncomp="2.2 kB" avg_uncomp="2.2 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-12T23:12:38.958473683Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e1e17907e%3A19e1e17e499%3A6b814d56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-12T23:12:39.411045174Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-12T23:12:39.411436263Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.61:11211 count=1
level=info ts=2026-05-12T23:12:39.427771047Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-12T23:12:39.427808894Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T23:12:39.427826633Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T23:12:39.42783885Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T23:12:39.427831952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:12:39.427929427Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000020
level=info ts=2026-05-12T23:12:39.427944937Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-12T23:12:39.427976571Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=25.526µs success=true
level=debug ts=2026-05-12T23:12:39.428183819Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=info ts=2026-05-12T23:12:39.861796685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.3 kB" avg_comp="5.3 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-12T23:12:39.864553631Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e1e176530%3A19e1e36fc0d%3A324e58f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-05-12T23:12:40.853593546Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T23:12:40.855263665Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-12T23:12:41.306619185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-12T23:12:41.309546671Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1e169c9b%3A19e1e1799f4%3A4197bdc9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-12T23:12:42.437723824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="30 kB" avg_comp="30 kB" total_uncomp="158 kB" avg_uncomp="158 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-12T23:12:42.440762682Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e1e16a129%3A19e1e1a011b%3A7b09cbb0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-12T23:12:42.761582834Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=warn ts=2026-05-12T23:12:44.427417663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:12:45.300067996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=10 total_comp="24 MB" avg_comp="2.4 MB" total_uncomp="198 MB" avg_uncomp="20 MB" full=8 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-12T23:12:45.307376587Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1e169cf0%3A19e1e1799d3%3Afb54655b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-12T23:12:45.777758489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9be421cde0f9c653 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="897 kB" avg_uncomp="897 kB" full=1 labels="{stream=\"stat-rep-felc\"}"
level=error ts=2026-05-12T23:12:45.780708145Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9be421cde0f9c653/19e1e285607%3A19e1e3051c9%3A2a80abf6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-felc\"}"
level=debug ts=2026-05-12T23:12:46.095674792Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-12T23:12:48.646228547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-12T23:12:48.648998173Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1e176530%3A19e1e18bce3%3Ae0b4d6ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-12T23:12:49.427941499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:12:49.429045265Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-05-12T23:12:49.648928823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7b03620a738d20e7 immediate=false num_chunks=1 total_comp="598 kB" avg_comp="598 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-lfwg\"}"
level=error ts=2026-05-12T23:12:49.653956732Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7b03620a738d20e7/19e1e28a71b%3A19e1e38b959%3A44ea025b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lfwg\"}"
level=info ts=2026-05-12T23:12:50.023559535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="424 kB" avg_comp="424 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-12T23:12:50.026967117Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e1e169c0f%3A19e1e528ea0%3Af15b888?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-05-12T23:12:50.371008005Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T23:12:52.368713025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:60470"
level=info ts=2026-05-12T23:12:52.455549952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="606 kB" avg_comp="606 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-12T23:12:52.459050918Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1e169a68%3A19e1e6ce0e9%3Ace8c9c84?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-12T23:12:52.761780068Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=info ts=2026-05-12T23:12:53.303309402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64bf9311b7fc1e3d immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-eolb\"}"
level=error ts=2026-05-12T23:12:53.30610845Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64bf9311b7fc1e3d/19e1e285dd8%3A19e1e3d2629%3Ad287ef44?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-eolb\"}"
level=info ts=2026-05-12T23:12:54.411920799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-12T23:12:54.415830602Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1e16a0d4%3A19e1e176531%3Aca245505?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-12T23:12:54.427964547Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:12:55.942337258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="458 kB" avg_comp="458 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-12T23:12:55.945777933Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e1e1695e0%3A19e1e6cda9a%3Ad92154c8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-12T23:12:56.095585261Z caller=checkpoint.go:372 msg="writing series" size="614 kB"
ts=2026-05-12T23:12:58.304349704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:45870"
level=warn ts=2026-05-12T23:12:59.427791283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:12:59.430512573Z caller=checkpoint.go:372 msg="writing series" size="677 kB"
level=debug ts=2026-05-12T23:12:59.430528987Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=7
level=debug ts=2026-05-12T23:13:02.762479505Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=warn ts=2026-05-12T23:13:04.427774921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:13:04.654730531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T23:13:04.660461899Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-05-12T23:13:04.6604833Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:13:04.660505851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T23:13:04.665344681Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-12T23:13:05.075210306Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-12T23:13:06.094896604Z caller=checkpoint.go:372 msg="writing series" size="894 B"
level=info ts=2026-05-12T23:13:06.751041159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T23:13:06.757402365Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T23:13:06.757421355Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:13:06.757441094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T23:13:06.7638499Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-12T23:13:08.381145686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=55884472be662265 immediate=false num_chunks=1 total_comp="605 kB" avg_comp="605 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" full=1 labels="{stream=\"konflux-bflt\"}"
level=error ts=2026-05-12T23:13:08.384907358Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/55884472be662265/19e1e28977b%3A19e1e36fc97%3Aa5385589?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-bflt\"}"
level=info ts=2026-05-12T23:13:08.530595901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=cf8cb9a5706fbae0 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-fqnw\"}"
level=error ts=2026-05-12T23:13:08.533383933Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/cf8cb9a5706fbae0/19e1e278ed0%3A19e1e325793%3Ac55b4380?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-fqnw\"}"
level=info ts=2026-05-12T23:13:09.427786579Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-12T23:13:09.42781243Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-12T23:13:09.427820379Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-12T23:13:09.427819186Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:13:09.428973061Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-12T23:13:12.442337722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="626 B" avg_comp="626 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T23:13:12.445468559Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T23:13:12.445488442Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-12T23:13:12.44551054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54fc5c40acb01431 immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gxyo\"}"
level=error ts=2026-05-12T23:13:12.447727611Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54fc5c40acb01431/19e1e279e70%3A19e1e27cc5c%3Aaede48f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gxyo\"}"
level=debug ts=2026-05-12T23:13:12.762786759Z caller=checkpoint.go:372 msg="writing series" size="663 kB"
level=debug ts=2026-05-12T23:13:12.762811191Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=warn ts=2026-05-12T23:13:14.427240657Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-12T23:13:14.70134406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="534 B" avg_comp="534 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-12T23:13:14.704113765Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e1e176530%3A19e1e176531%3Ad630e56e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-12T23:13:16.095732921Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=info ts=2026-05-12T23:13:16.448923486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="502 kB" avg_comp="502 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-12T23:13:16.452525248Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1e1695df%3A19e1e42c52a%3A3726ad42?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-12T23:13:17.563405741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-12T23:13:17.568354547Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1e16a0e9%3A19e1e5a2563%3Acb1752f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-12T23:13:18.129910002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-12T23:13:18.132808606Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e1e176530%3A19e1e32a1cb%3A1bac6966?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-12T23:13:19.427317618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-12T23:13:19.428408261Z caller=checkpoint.go:372 msg="writing series" size="1.0 kB"
level=info ts=2026-05-12T23:13:19.996052888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-12T23:13:20.00253531Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e1e176530%3A19e1e497802%3A5914346c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
ts=2026-05-12T23:13:20.37381077Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f2155d35 10.130.0.61:7946"
ts=2026-05-12T23:13:21.413532639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.59:47170"
level=debug ts=2026-05-12T23:13:22.762294228Z caller=checkpoint.go:372 msg="writing series" size="813 kB"
level=info ts=2026-05-12T23:13:23.842108857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-12T23:13:23.849734656Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1e169676%3A19e1e3a2c40%3A46fb8780?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"