level=info ts=2026-04-29T19:18:13.079281067Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-04-29T19:18:13.07930648Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-04-29T19:18:13.079639167Z caller=modules.go:2203 msg="initializing ingester query tags interceptors"
level=info ts=2026-04-29T19:18:13.080495239Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-29T19:18:13.080925532Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-29T19:18:13.081002566Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.63 inf=eth0
level=info ts=2026-04-29T19:18:13.081554327Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-e127a7f3
ts=2026-04-29T19:18:13.081634742Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-29T19:18:13.081649536Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.63 advertisePort=7946
level=debug ts=2026-04-29T19:18:13.081665885Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.63 advertisePort=7946
level=info ts=2026-04-29T19:18:13.081781422Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-04-29T19:18:13.086469229Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-04-29T19:18:13.086503307Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
level=info ts=2026-04-29T19:18:13.087802888Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-04-29T19:18:13.087844383Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
ts=2026-04-29T19:18:13.088062095Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-29T19:18:13.088074788Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-29T19:18:13.088084958Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.305477ms
level=error ts=2026-04-29T19:18:13.088098849Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-04-29T19:18:13.088105858Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-04-29T19:18:13.088215736Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-04-29T19:18:13.08825069Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-29T19:18:13.08824546Z
level=debug ts=2026-04-29T19:18:13.088272554Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-04-29T19:18:13.088310843Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-04-29T19:18:13.088332903Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-04-29T19:18:13.088348529Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-29T19:18:13.088346916Z
level=debug ts=2026-04-29T19:18:13.088364529Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=debug ts=2026-04-29T19:18:13.090352383Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-29T19:18:13.090461126Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.129.0.63 inf=eth0
ts=2026-04-29T19:18:13.090632513Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T19:18:13.090646382Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-29T19:18:13.108319453Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=info ts=2026-04-29T19:18:13.108321516Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-29T19:18:13.108326304Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=info ts=2026-04-29T19:18:13.108347736Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-29T19:18:13.108361797Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-04-29T19:18:13.108337031Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-04-29T19:18:13.108386586Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-29T19:18:13.108401848Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-29T19:18:13.108421069Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-04-29T19:18:13.108422951Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-04-29T19:18:13.108871359Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=debug ts=2026-04-29T19:18:13.108874093Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-04-29T19:18:13.108880201Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-04-29T19:18:13.108881734Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-04-29T19:18:13.108892801Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=info ts=2026-04-29T19:18:13.108915966Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-04-29T19:18:13.108941037Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-04-29T19:18:13.108945457Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=debug ts=2026-04-29T19:18:13.108948432Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=info ts=2026-04-29T19:18:13.108951119Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-29T19:18:13.109065137Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-04-29T19:18:13.109071227Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-04-29T19:18:13.109074592Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-04-29T19:18:13.10907765Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=info ts=2026-04-29T19:18:13.109080031Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-04-29T19:18:13.109090394Z caller=ingester.go:571 component=ingester msg="recovering from checkpoint"
level=info ts=2026-04-29T19:18:13.10910069Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-29T19:18:13.109150765Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=27.138µs success=true
level=info ts=2026-04-29T19:18:13.109163566Z caller=recovery.go:45 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-04-29T19:18:13.109210654Z caller=ingester.go:587 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=121.596µs errors=false
level=info ts=2026-04-29T19:18:13.10921961Z caller=ingester.go:593 component=ingester msg="recovering from WAL"
level=info ts=2026-04-29T19:18:13.10933838Z caller=ingester.go:609 component=ingester msg="WAL segment recovery finished" elapsed=249.524µs errors=false
level=info ts=2026-04-29T19:18:13.109345777Z caller=ingester.go:557 component=ingester msg="closing recoverer"
level=info ts=2026-04-29T19:18:13.109351396Z caller=ingester.go:565 component=ingester msg="WAL recovery finished" time=262.422µs
level=info ts=2026-04-29T19:18:13.109369273Z caller=wal.go:173 msg=started component=wal
level=info ts=2026-04-29T19:18:13.109692592Z caller=lifecycler.go:694 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-29T19:18:13.109693438Z caller=ingester.go:778 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=17.688692508s
level=info ts=2026-04-29T19:18:13.109719941Z caller=lifecycler.go:721 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-04-29T19:18:13.109784467Z caller=ingester.go:473 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-04-29T19:18:13.109789516Z caller=loki.go:615 msg="Loki started" startup_time=91.157466ms
level=debug ts=2026-04-29T19:18:13.109807389Z caller=lifecycler.go:558 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-04-29T19:18:13.109813503Z caller=lifecycler.go:562 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=debug ts=2026-04-29T19:18:13.109852083Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-04-29T19:18:14.411567333Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T19:18:14.411591038Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T19:18:17.05455261Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T19:18:17.054575353Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-29T19:18:18.109932477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:18:23.109854586Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:18:23.497171691Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T19:18:23.497193906Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-29T19:18:28.110797699Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:18:28.634029665Z caller=logging.go:137 msg="GET /ready (503) 95.552µs"
ts=2026-04-29T19:18:32.106611958Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T19:18:32.1066348Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-29T19:18:33.109839259Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:18:38.110778327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:18:38.631798446Z caller=logging.go:137 msg="GET /ready (503) 70.774µs"
level=info ts=2026-04-29T19:18:43.110737248Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-29T19:18:43.110779885Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-04-29T19:18:43.110784996Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:18:43.110793897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:18:48.110030161Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:18:53.109849704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:18:53.51573773Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.63:7946"
ts=2026-04-29T19:18:53.515854941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.63:45990"
ts=2026-04-29T19:18:53.518360856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.81:7946"
ts=2026-04-29T19:18:53.521280376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.79:7946"
ts=2026-04-29T19:18:53.523592317Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.78:7946: dial tcp 10.128.0.78:7946: connect: connection refused"
ts=2026-04-29T19:18:53.524862118Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.80:7946: dial tcp 10.128.0.80:7946: connect: connection refused"
ts=2026-04-29T19:18:53.527296915Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.78:7946: dial tcp 10.130.0.78:7946: connect: connection refused"
ts=2026-04-29T19:18:53.528789034Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.79:7946"
level=info ts=2026-04-29T19:18:53.530315961Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=4 elapsed_time=40.442205571s
ts=2026-04-29T19:18:54.579532514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42004"
ts=2026-04-29T19:18:55.53859474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:58760"
level=warn ts=2026-04-29T19:18:58.110051608Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:18:59.79637421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:43448"
level=debug ts=2026-04-29T19:19:01.240860788Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T19:19:03.110824811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:19:08.110150121Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:19:12.165352143Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-29T19:19:13.087917352Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:19:13.088225924Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:19:13.11058079Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T19:19:13.110596663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:19:13.110608665Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:19:13.110619297Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-29T19:19:13.638426961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:38762"
ts=2026-04-29T19:19:15.886344297Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=warn ts=2026-04-29T19:19:18.110157126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:19:23.110690557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:19:28.110618823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:19:28.369566551Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-29T19:19:33.109868911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:19:33.789187285Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T19:19:33.789233124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:19:33.797109414Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-29T19:19:36.600875889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:46010"
level=warn ts=2026-04-29T19:19:38.110447103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:19:43.110453356Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:19:43.110485692Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:19:43.110497571Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:19:43.11054324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:19:45.889568832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T19:19:46.043867816Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-29T19:19:48.110690526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:19:49.261224492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:19:49.268383638Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T19:19:53.109965316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:19:58.110352696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:20:03.110716229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:20:07.741541351Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T19:20:08.110781034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:20:11.259182236Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-29T19:20:13.087949613Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:20:13.088144669Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:20:13.110508071Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:20:13.110536561Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:20:13.110544423Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:20:13.110542211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:20:15.892872968Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=warn ts=2026-04-29T19:20:18.110631957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:20:23.110771288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:20:28.110788891Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:20:28.928194578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:20:28.934938317Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T19:20:33.110042061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:20:34.136182921Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-29T19:20:36.605308467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:45414"
level=debug ts=2026-04-29T19:20:37.553355561Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-29T19:20:37.555098885Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T19:20:38.110492946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:20:43.110690261Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:20:43.110727039Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:20:43.110734954Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:20:43.110769439Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:20:45.895601262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=warn ts=2026-04-29T19:20:48.110864383Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:20:53.110864427Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:20:58.110495559Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:20:59.468875966Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T19:21:03.110787796Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:21:06.607997481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:59944"
level=warn ts=2026-04-29T19:21:08.110468802Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:21:13.088089082Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:21:13.088124957Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:21:13.110565753Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T19:21:13.11057668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:21:13.110592506Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:21:13.110600599Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T19:21:14.602401147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:21:14.609857316Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T19:21:15.555862084Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T19:21:15.898060433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=warn ts=2026-04-29T19:21:18.110668035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:21:23.110398381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:21:24.681205323Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T19:21:28.110682876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:21:33.110419293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:21:38.110365511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:21:43.110342529Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:21:43.110372508Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-29T19:21:43.110361512Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:21:43.110381043Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-29T19:21:43.651321167Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:58854"
ts=2026-04-29T19:21:45.900724641Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=warn ts=2026-04-29T19:21:48.110772959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:21:49.186232635Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T19:21:53.110013069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:21:56.32598271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:21:56.333137181Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T19:21:58.110560438Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:22:03.110590963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:22:08.110148476Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:22:13.088209193Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:22:13.088723052Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:22:13.110007452Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:22:13.110038012Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:22:13.110047621Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:22:13.110044738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:22:15.903369176Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T19:22:16.642202747Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-29T19:22:18.110327444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:22:23.110361953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:22:24.033372984Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T19:22:28.11056061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:22:29.81200108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:36514"
level=warn ts=2026-04-29T19:22:33.110291224Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:22:36.61301932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:34920"
level=warn ts=2026-04-29T19:22:38.110812056Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:22:43.110365437Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:22:43.110405697Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:22:43.110414305Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:22:43.110464779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:22:43.894460642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:22:43.9035561Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-29T19:22:45.906189605Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=warn ts=2026-04-29T19:22:48.110484569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:22:53.110463095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:22:53.266884952Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T19:22:58.11038418Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:23:03.110173992Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:23:08.11056579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:23:13.088098288Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:23:13.088574233Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:23:13.109831782Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-29T19:23:13.109853878Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-04-29T19:23:13.109870146Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:23:13.109878537Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:23:13.109889379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:23:13.109987924Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=info ts=2026-04-29T19:23:13.109996028Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-29T19:23:13.110009401Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=9.185µs success=true
level=debug ts=2026-04-29T19:23:13.110198138Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
ts=2026-04-29T19:23:15.909420139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T19:23:17.465904383Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=warn ts=2026-04-29T19:23:18.110635781Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:23:18.556075006Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:23:21.819938696Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-04-29T19:23:23.109945416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:23:26.175082568Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-04-29T19:23:28.110370521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:23:28.784429585Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-29T19:23:30.530391482Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=warn ts=2026-04-29T19:23:33.110238475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:23:34.885404931Z caller=checkpoint.go:372 msg="writing series" size="1.6 kB"
level=warn ts=2026-04-29T19:23:38.109841899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:23:39.239936697Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-04-29T19:23:41.060545724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:23:41.068677808Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T19:23:43.109993669Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:23:43.110024532Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:23:43.110032868Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:23:43.110067503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:23:43.595909697Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-04-29T19:23:43.595932715Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=8
ts=2026-04-29T19:23:45.912513931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T19:23:47.949626751Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-04-29T19:23:48.110156233Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:23:49.126819864Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:23:52.304831451Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=warn ts=2026-04-29T19:23:53.110667467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:23:56.658975451Z caller=checkpoint.go:372 msg="writing series" size="229 kB"
level=warn ts=2026-04-29T19:23:58.110346796Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:24:01.014211934Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-29T19:24:03.110611386Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:24:05.368762943Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=warn ts=2026-04-29T19:24:08.110500146Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:24:09.72396712Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-04-29T19:24:13.088866353Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:24:13.089656565Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:24:13.109823729Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:24:13.109841666Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:24:13.109848954Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:24:13.109911373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:24:14.078926088Z caller=checkpoint.go:372 msg="writing series" size="268 kB"
ts=2026-04-29T19:24:15.914688936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T19:24:16.417546949Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T19:24:18.110172787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:24:18.43361834Z caller=checkpoint.go:372 msg="writing series" size="40 kB"
level=debug ts=2026-04-29T19:24:22.788426333Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-04-29T19:24:23.110706591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:24:27.143334722Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=warn ts=2026-04-29T19:24:28.110042826Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:24:30.065721538Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-29T19:24:31.49816569Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=warn ts=2026-04-29T19:24:33.110784488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:24:35.852582955Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
ts=2026-04-29T19:24:36.621340783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:53164"
level=info ts=2026-04-29T19:24:37.988398008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:24:37.995460707Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T19:24:38.110190071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:24:40.207322694Z caller=checkpoint.go:372 msg="writing series" size="274 kB"
level=info ts=2026-04-29T19:24:43.110133663Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:24:43.110164677Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:24:43.110172206Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:24:43.110226946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:24:43.665989537Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36930"
level=debug ts=2026-04-29T19:24:44.562619317Z caller=checkpoint.go:372 msg="writing series" size="213 kB"
level=debug ts=2026-04-29T19:24:44.56264069Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=14
level=debug ts=2026-04-29T19:24:45.376075548Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-29T19:24:45.918026422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=warn ts=2026-04-29T19:24:48.110805718Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:24:48.916751357Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=warn ts=2026-04-29T19:24:53.109867531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:24:53.272284234Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=debug ts=2026-04-29T19:24:57.626731668Z caller=checkpoint.go:372 msg="writing series" size="8.3 kB"
level=warn ts=2026-04-29T19:24:58.110165873Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:24:59.822779889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:42968"
level=debug ts=2026-04-29T19:25:01.98178419Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=warn ts=2026-04-29T19:25:03.110406561Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:25:06.336519752Z caller=checkpoint.go:372 msg="writing series" size="9.3 kB"
level=warn ts=2026-04-29T19:25:08.110474687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:25:10.691064949Z caller=checkpoint.go:372 msg="writing series" size="204 kB"
level=info ts=2026-04-29T19:25:13.088258609Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:25:13.089243312Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:25:13.110513023Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:25:13.110535748Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:25:13.110544054Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:25:13.11055703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:25:15.046158766Z caller=checkpoint.go:372 msg="writing series" size="513 B"
ts=2026-04-29T19:25:15.920842184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=warn ts=2026-04-29T19:25:18.112505111Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:25:19.400988543Z caller=checkpoint.go:372 msg="writing series" size="439 kB"
level=debug ts=2026-04-29T19:25:19.621721938Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T19:25:21.254237077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:25:21.262390475Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T19:25:23.110424859Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:25:23.755574046Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=warn ts=2026-04-29T19:25:28.110788551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:25:28.110833978Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-04-29T19:25:32.466058617Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-04-29T19:25:33.11035681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:25:36.820248799Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-04-29T19:25:38.110671725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:25:38.186422476Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-29T19:25:38.188191986Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-29T19:25:41.175020569Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=info ts=2026-04-29T19:25:43.110618309Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T19:25:43.110625785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:25:43.110647004Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:25:43.110654883Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T19:25:45.530023339Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
ts=2026-04-29T19:25:45.92314298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=warn ts=2026-04-29T19:25:48.110891619Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:25:48.858383935Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:25:49.884611108Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=debug ts=2026-04-29T19:25:49.884634188Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=15
level=warn ts=2026-04-29T19:25:53.110340774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:25:54.240230057Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=warn ts=2026-04-29T19:25:58.110166262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:25:58.594346925Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
ts=2026-04-29T19:25:59.828523378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:53986"
level=debug ts=2026-04-29T19:26:02.949999385Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=warn ts=2026-04-29T19:26:03.110524542Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:26:05.429970502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:26:05.439028909Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:26:05.439052284Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T19:26:05.439075829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:26:05.445270536Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T19:26:07.304671644Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=warn ts=2026-04-29T19:26:08.110072841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:26:11.661971233Z caller=checkpoint.go:372 msg="writing series" size="5.1 MB"
level=debug ts=2026-04-29T19:26:11.66199504Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.6 MB" series=5
level=info ts=2026-04-29T19:26:13.087944328Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:26:13.088410889Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:26:13.110780887Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:26:13.110800684Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-29T19:26:13.11079407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:26:13.110809038Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-29T19:26:13.671706568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:40208"
ts=2026-04-29T19:26:15.925512748Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T19:26:16.014038537Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=warn ts=2026-04-29T19:26:18.110555296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:26:18.651219958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:26:18.65894043Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T19:26:20.369236442Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-04-29T19:26:23.110642559Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:26:24.03751732Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:26:24.723134196Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-04-29T19:26:28.11061931Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:26:29.078258364Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-04-29T19:26:33.110798125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:26:33.433794139Z caller=checkpoint.go:372 msg="writing series" size="355 kB"
level=debug ts=2026-04-29T19:26:34.573845511Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:26:36.62992462Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:43382"
level=debug ts=2026-04-29T19:26:37.788393814Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=warn ts=2026-04-29T19:26:38.110274191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:26:42.143209244Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
level=info ts=2026-04-29T19:26:43.110278641Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:26:43.11031003Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:26:43.110317799Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:26:43.110312454Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:26:45.928309735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T19:26:46.498014301Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=warn ts=2026-04-29T19:26:48.109921766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:26:50.85237592Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-04-29T19:26:53.110242691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:26:54.590137719Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T19:26:54.640903406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:26:54.650464684Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T19:26:55.207268291Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-04-29T19:26:58.110422467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:26:59.562235478Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=warn ts=2026-04-29T19:27:03.110451324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:27:03.917424909Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-29T19:27:08.110630364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:27:08.272074228Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=debug ts=2026-04-29T19:27:12.627180588Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=debug ts=2026-04-29T19:27:12.627207179Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=14
level=info ts=2026-04-29T19:27:13.088338145Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:27:13.089050087Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:27:13.110417967Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:27:13.110445475Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:27:13.110454495Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:27:13.110467014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:27:15.931498524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T19:27:16.981701275Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-29T19:27:18.110202541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:27:21.336248712Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-04-29T19:27:23.109908021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:27:23.440750591Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:27:24.669124757Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T19:27:25.691330018Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-04-29T19:27:28.110738561Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:27:30.046650082Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=warn ts=2026-04-29T19:27:33.109893964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:27:34.401576813Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
ts=2026-04-29T19:27:36.634981795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:60568"
level=warn ts=2026-04-29T19:27:38.110326054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:27:38.755860301Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=info ts=2026-04-29T19:27:41.38263548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:27:41.390013802Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T19:27:43.110253226Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T19:27:43.110256317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:27:43.11028405Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:27:43.110293732Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T19:27:43.110282389Z caller=checkpoint.go:384 msg="flushing series" totalSize="561 kB" series=6
level=info ts=2026-04-29T19:27:43.116313327Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-04-29T19:27:43.119570331Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009427425s
ts=2026-04-29T19:27:45.934311373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=warn ts=2026-04-29T19:27:48.11015164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:27:48.607814624Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T19:27:53.109999957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:27:58.109894227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:28:03.109959778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:28:05.170829579Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:28:06.637604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:58688"
level=warn ts=2026-04-29T19:28:08.109963352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:28:13.088033309Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:28:13.088150799Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:28:13.109556584Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-29T19:28:13.10972617Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=info ts=2026-04-29T19:28:13.109736632Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-29T19:28:13.109757563Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=16.035µs success=true
level=info ts=2026-04-29T19:28:13.109815565Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:28:13.109842914Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:28:13.109860998Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T19:28:13.109908976Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-04-29T19:28:13.109955734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:28:15.937004426Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T19:28:17.46542276Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=warn ts=2026-04-29T19:28:18.110089579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:28:21.820317753Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-04-29T19:28:23.110225458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:28:24.219884986Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:28:26.175423908Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-04-29T19:28:28.11085661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:28:30.530323948Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=info ts=2026-04-29T19:28:30.811103706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:28:30.819592857Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T19:28:33.110370433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:28:34.884262938Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
level=warn ts=2026-04-29T19:28:38.110615419Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:28:39.239621452Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=info ts=2026-04-29T19:28:43.110251233Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:28:43.110282353Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:28:43.110290843Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:28:43.110296995Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:28:43.594690432Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-29T19:28:43.594711403Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=8
ts=2026-04-29T19:28:45.939243397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T19:28:47.949141572Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-04-29T19:28:48.110690048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:28:52.304064454Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=warn ts=2026-04-29T19:28:53.109880048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:28:53.591608738Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:28:56.65928134Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=warn ts=2026-04-29T19:28:58.110539426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:29:01.013732104Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-04-29T19:29:01.384886535Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-29T19:29:03.110155892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:29:05.368504277Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-29T19:29:08.110035159Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:29:09.723545443Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-04-29T19:29:13.088362422Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:29:13.089177712Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:29:13.110639095Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:29:13.110670512Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:29:13.110680173Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:29:13.110677959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:29:13.685305878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:37136"
level=debug ts=2026-04-29T19:29:14.077845693Z caller=checkpoint.go:372 msg="writing series" size="268 kB"
level=info ts=2026-04-29T19:29:15.806488556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:29:15.814496539Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-29T19:29:15.941996415Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=warn ts=2026-04-29T19:29:18.110373688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:29:18.433262928Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
level=debug ts=2026-04-29T19:29:22.788084203Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-04-29T19:29:23.11075278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:29:25.392761383Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:29:27.142677225Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-04-29T19:29:28.110367417Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:29:31.498032957Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=warn ts=2026-04-29T19:29:33.110057442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:29:35.852268421Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-29T19:29:38.110696268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:29:40.207615842Z caller=checkpoint.go:372 msg="writing series" size="176 kB"
level=info ts=2026-04-29T19:29:43.110007775Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:29:43.110041048Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:29:43.110048681Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:29:43.110051168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:29:44.562401923Z caller=checkpoint.go:372 msg="writing series" size="319 kB"
level=debug ts=2026-04-29T19:29:44.562430685Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=14
ts=2026-04-29T19:29:45.944806643Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=warn ts=2026-04-29T19:29:48.110376801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:29:48.762194577Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T19:29:48.764000889Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-29T19:29:48.916679802Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-04-29T19:29:53.110498399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:29:53.271798537Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=debug ts=2026-04-29T19:29:54.58627103Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:29:57.626324516Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
level=warn ts=2026-04-29T19:29:58.110731165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:29:59.847627142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:48136"
level=debug ts=2026-04-29T19:30:01.981862275Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=warn ts=2026-04-29T19:30:03.109952698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:30:06.336050566Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
ts=2026-04-29T19:30:06.646089279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:60292"
level=warn ts=2026-04-29T19:30:08.110477667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:30:08.899467865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:30:08.907127513Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T19:30:10.691387895Z caller=checkpoint.go:372 msg="writing series" size="234 kB"
level=info ts=2026-04-29T19:30:13.091508155Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:30:13.091815254Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-29T19:30:13.091877012Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974989 n=1
level=debug ts=2026-04-29T19:30:13.091936394Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974989 n=1
level=debug ts=2026-04-29T19:30:13.092896853Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=warn ts=2026-04-29T19:30:13.110173884Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:30:13.110215646Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:30:13.110236442Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:30:13.110248049Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T19:30:15.045775678Z caller=checkpoint.go:372 msg="writing series" size="513 B"
ts=2026-04-29T19:30:15.947786481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=warn ts=2026-04-29T19:30:18.110380969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:30:19.401464891Z caller=checkpoint.go:372 msg="writing series" size="537 kB"
level=debug ts=2026-04-29T19:30:19.401488884Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-04-29T19:30:23.110772201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:30:23.755229858Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-04-29T19:30:28.110310988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:30:28.110386206Z caller=checkpoint.go:372 msg="writing series" size="83 kB"
level=debug ts=2026-04-29T19:30:30.24550896Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T19:30:30.245560126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:30:30.254561492Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T19:30:32.465056708Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=warn ts=2026-04-29T19:30:33.110284399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:30:36.81996012Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-04-29T19:30:38.110656201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:30:41.175357717Z caller=checkpoint.go:372 msg="writing series" size="265 kB"
level=info ts=2026-04-29T19:30:43.109931104Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:30:43.109963457Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:30:43.109973611Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:30:43.109973172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:30:45.530084823Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
ts=2026-04-29T19:30:45.950490375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=warn ts=2026-04-29T19:30:48.11055359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:30:49.07468279Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-29T19:30:49.297304596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:30:49.303870451Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T19:30:49.884361589Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=warn ts=2026-04-29T19:30:53.110314082Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:30:54.239241151Z caller=checkpoint.go:372 msg="writing series" size="359 kB"
level=warn ts=2026-04-29T19:30:58.110011437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:30:58.594742019Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=debug ts=2026-04-29T19:31:02.949632602Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-04-29T19:31:02.949658999Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-04-29T19:31:03.11062901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:31:05.277492929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:31:05.287287819Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T19:31:05.763553493Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:31:07.304558783Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=warn ts=2026-04-29T19:31:08.110702492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:31:09.403827083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.8 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:31:09.412023083Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T19:31:11.662981455Z caller=checkpoint.go:372 msg="writing series" size="5.9 MB"
level=debug ts=2026-04-29T19:31:11.663009573Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.1 MB" series=2
level=info ts=2026-04-29T19:31:13.087921928Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:31:13.088381735Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:31:13.109830892Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:31:13.109870223Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:31:13.109878896Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:31:13.109883618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:31:15.954144313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T19:31:16.014154517Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=warn ts=2026-04-29T19:31:18.110301342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:31:20.368507517Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-04-29T19:31:23.110374061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:31:24.723277318Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=warn ts=2026-04-29T19:31:28.110308056Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:31:29.078513801Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-04-29T19:31:33.110202183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:31:33.432959283Z caller=checkpoint.go:372 msg="writing series" size="279 kB"
level=debug ts=2026-04-29T19:31:37.788334819Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
level=warn ts=2026-04-29T19:31:38.110353686Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:31:38.823087686Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:31:42.14234395Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
level=info ts=2026-04-29T19:31:43.110578217Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T19:31:43.110591679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:31:43.110614497Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:31:43.110623923Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-29T19:31:45.957057693Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T19:31:46.497727712Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=warn ts=2026-04-29T19:31:48.110279758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:31:50.433100239Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T19:31:50.852475585Z caller=checkpoint.go:372 msg="writing series" size="398 kB"
level=debug ts=2026-04-29T19:31:50.852499104Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=warn ts=2026-04-29T19:31:53.110156863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:31:55.207686941Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-04-29T19:31:58.110706744Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:31:59.562280484Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=info ts=2026-04-29T19:32:00.13065361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.7 MB" avg_comp="2.9 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:32:00.139634984Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T19:32:03.110336896Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:32:03.578754644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:32:03.585625931Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T19:32:03.916666737Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
ts=2026-04-29T19:32:06.656522609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:53772"
level=warn ts=2026-04-29T19:32:08.110231832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:32:08.271666304Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=debug ts=2026-04-29T19:32:08.436556826Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:32:12.626528292Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=info ts=2026-04-29T19:32:13.088175794Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:32:13.088190435Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:32:13.111731821Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T19:32:13.11174119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:32:13.111764268Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:32:13.111775063Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-29T19:32:13.697858037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:56338"
ts=2026-04-29T19:32:15.959765165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T19:32:16.981255039Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-29T19:32:18.110302443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:32:21.336320416Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-04-29T19:32:23.110855406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:32:25.691061335Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-04-29T19:32:28.110840601Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:32:29.859640926Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:37178"
level=debug ts=2026-04-29T19:32:30.045936139Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
level=debug ts=2026-04-29T19:32:30.04596145Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-04-29T19:32:33.110672245Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:32:34.401118071Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=warn ts=2026-04-29T19:32:38.110406706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:32:38.763241197Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-04-29T19:32:38.763262239Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=2
level=debug ts=2026-04-29T19:32:40.291421274Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T19:32:43.110405594Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:32:43.110424311Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:32:43.110466064Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:32:43.110474546Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T19:32:43.123039405Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-04-29T19:32:43.124485249Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.014613269s
ts=2026-04-29T19:32:43.700730859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36480"
ts=2026-04-29T19:32:45.962839434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=info ts=2026-04-29T19:32:47.70471329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:32:47.715971722Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T19:32:48.110050873Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:32:52.199121993Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-29T19:32:53.110735183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:32:58.110705431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:32:59.145447896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:32:59.153667639Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:32:59.153694622Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T19:32:59.153723404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:32:59.16068132Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T19:33:03.110287514Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:33:06.660185009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:38830"
level=debug ts=2026-04-29T19:33:08.010136223Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T19:33:08.110471308Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:33:13.088870213Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:33:13.089538878Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:33:13.109852974Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-04-29T19:33:13.109926464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:33:13.109922796Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:33:13.109952691Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:33:13.109962212Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T19:33:13.109995918Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=info ts=2026-04-29T19:33:13.110005521Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-29T19:33:13.110024171Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.558µs success=true
level=debug ts=2026-04-29T19:33:13.110193858Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=info ts=2026-04-29T19:33:13.43720518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:33:13.445747634Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-29T19:33:13.704508985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:53130"
ts=2026-04-29T19:33:15.965478582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T19:33:16.572615746Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=warn ts=2026-04-29T19:33:18.110287637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:33:20.03397533Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-04-29T19:33:23.110216174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:33:23.495570648Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=debug ts=2026-04-29T19:33:26.957081124Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-04-29T19:33:28.110010219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:33:30.418953222Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=info ts=2026-04-29T19:33:32.49214044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:33:32.505418669Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T19:33:33.110464565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:33:33.879748225Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=debug ts=2026-04-29T19:33:36.566307231Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-29T19:33:36.663142857Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:55752"
level=debug ts=2026-04-29T19:33:37.341587639Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=warn ts=2026-04-29T19:33:38.110907351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:33:40.803676029Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-04-29T19:33:40.803700605Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=9
level=info ts=2026-04-29T19:33:43.110338099Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:33:43.110374732Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:33:43.110387187Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:33:43.11039525Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:33:43.707766393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:52556"
level=debug ts=2026-04-29T19:33:44.264949067Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=debug ts=2026-04-29T19:33:44.645911356Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:33:45.968281863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T19:33:47.725919229Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=warn ts=2026-04-29T19:33:48.110478177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:33:51.187813692Z caller=checkpoint.go:372 msg="writing series" size="441 kB"
level=info ts=2026-04-29T19:33:53.048067866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:33:53.055677857Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T19:33:53.110240107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:33:54.648944598Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-04-29T19:33:58.110678704Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=warn ts=2026-04-29T19:33:58.110689658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:34:01.572391464Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-04-29T19:34:01.647735024Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T19:34:03.110250225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:34:05.034029791Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
ts=2026-04-29T19:34:06.66632274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:50038"
level=warn ts=2026-04-29T19:34:08.110400042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:34:08.495375757Z caller=checkpoint.go:372 msg="writing series" size="268 kB"
level=debug ts=2026-04-29T19:34:11.957123824Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
level=info ts=2026-04-29T19:34:12.802363427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:34:12.814112875Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T19:34:13.088241779Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:34:13.088921073Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:34:13.110427805Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:34:13.110479481Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:34:13.110487735Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:34:13.110485491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:34:15.41857911Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
ts=2026-04-29T19:34:15.971104658Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=warn ts=2026-04-29T19:34:18.110305505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:34:18.880338529Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=debug ts=2026-04-29T19:34:18.880362832Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=debug ts=2026-04-29T19:34:22.341731216Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-04-29T19:34:23.1099234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:34:25.803237205Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=warn ts=2026-04-29T19:34:28.110866375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:34:29.264376583Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-04-29T19:34:32.72577311Z caller=checkpoint.go:372 msg="writing series" size="299 kB"
level=warn ts=2026-04-29T19:34:33.110218296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:34:34.877652188Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T19:34:34.941082731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:34:34.948098331Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T19:34:35.116787273Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T19:34:35.118943226Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-29T19:34:36.187998735Z caller=checkpoint.go:372 msg="writing series" size="238 kB"
level=warn ts=2026-04-29T19:34:38.110091003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:34:39.648955998Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=info ts=2026-04-29T19:34:43.110171939Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:34:43.110202886Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:34:43.110221211Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T19:34:43.110219078Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-29T19:34:43.110221031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:34:45.973969359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T19:34:46.572105058Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=warn ts=2026-04-29T19:34:48.110971415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:34:50.033977846Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=warn ts=2026-04-29T19:34:53.11052241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:34:53.239827024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:34:53.260829991Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T19:34:53.495507816Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=debug ts=2026-04-29T19:34:56.957454826Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-04-29T19:34:58.110622361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:34:58.992417006Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:35:00.418209541Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-04-29T19:35:03.11054686Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:35:03.880036334Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=debug ts=2026-04-29T19:35:03.880063044Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=13
ts=2026-04-29T19:35:06.671762096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:42078"
level=debug ts=2026-04-29T19:35:07.341458481Z caller=checkpoint.go:372 msg="writing series" size="513 B"
level=warn ts=2026-04-29T19:35:08.110598927Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:35:10.803273859Z caller=checkpoint.go:372 msg="writing series" size="658 kB"
level=info ts=2026-04-29T19:35:13.088913515Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:35:13.089724758Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:35:13.110130334Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T19:35:13.110132373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:35:13.110156187Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:35:13.110166205Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T19:35:14.264419488Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
ts=2026-04-29T19:35:15.977267902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T19:35:17.72604232Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-04-29T19:35:18.110408268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:35:21.187475978Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=warn ts=2026-04-29T19:35:23.110252453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:35:24.648899379Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=debug ts=2026-04-29T19:35:26.565554215Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T19:35:28.110464647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:35:28.110500099Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=debug ts=2026-04-29T19:35:29.103144867Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-29T19:35:30.829212409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:35:30.836268915Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T19:35:31.572492173Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-04-29T19:35:33.110116941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:35:35.033935863Z caller=checkpoint.go:372 msg="writing series" size="305 kB"
level=debug ts=2026-04-29T19:35:35.033957321Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=info ts=2026-04-29T19:35:37.655793227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:35:37.665395037Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T19:35:38.110026818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:35:38.495041744Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-04-29T19:35:41.956789208Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=info ts=2026-04-29T19:35:43.11057272Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:35:43.110604721Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:35:43.110613287Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:35:43.110647507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:35:45.41826038Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
ts=2026-04-29T19:35:45.98091368Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=warn ts=2026-04-29T19:35:48.110396291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:35:48.880413371Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=debug ts=2026-04-29T19:35:52.341611744Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=warn ts=2026-04-29T19:35:53.109902522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:35:55.803410511Z caller=checkpoint.go:372 msg="writing series" size="176 kB"
level=warn ts=2026-04-29T19:35:58.110221862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:35:59.268931466Z caller=checkpoint.go:372 msg="writing series" size="6.4 MB"
level=debug ts=2026-04-29T19:35:59.268964993Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.3 MB" series=7
ts=2026-04-29T19:35:59.877638468Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:49380"
level=debug ts=2026-04-29T19:36:00.596914377Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:36:02.726171682Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=warn ts=2026-04-29T19:36:03.110724605Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:36:06.187992081Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=warn ts=2026-04-29T19:36:08.110220425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:36:09.649204384Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=info ts=2026-04-29T19:36:13.088304315Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:36:13.088970388Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:36:13.110480003Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:36:13.110506Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:36:13.110514377Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T19:36:13.110508194Z caller=checkpoint.go:372 msg="writing series" size="8.8 kB"
level=warn ts=2026-04-29T19:36:13.110517847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:36:13.719328867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:46742"
level=info ts=2026-04-29T19:36:15.896414689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:36:15.904179602Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-29T19:36:15.983484343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T19:36:16.571791897Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=debug ts=2026-04-29T19:36:18.107039688Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-29T19:36:18.11004844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:36:20.034129775Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-04-29T19:36:23.109867644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:36:23.496020632Z caller=checkpoint.go:372 msg="writing series" size="559 kB"
level=debug ts=2026-04-29T19:36:26.957305399Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=warn ts=2026-04-29T19:36:28.110638696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:36:29.88132866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:43636"
level=debug ts=2026-04-29T19:36:30.418210232Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-04-29T19:36:33.110812633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:36:33.880413517Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
level=debug ts=2026-04-29T19:36:33.880452202Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=debug ts=2026-04-29T19:36:36.556346478Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T19:36:36.908399351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:36:36.916128875Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:36:36.916148166Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T19:36:36.916182366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:36:36.922120843Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T19:36:37.341662809Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=warn ts=2026-04-29T19:36:38.110575999Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:36:40.803554503Z caller=checkpoint.go:372 msg="writing series" size="466 kB"
level=info ts=2026-04-29T19:36:43.110201231Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:36:43.110230578Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:36:43.110240361Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:36:43.110267093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:36:44.265108308Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
ts=2026-04-29T19:36:45.985888022Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T19:36:47.726349117Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-04-29T19:36:48.11047282Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:36:50.234632813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:36:50.24565674Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T19:36:51.18746565Z caller=checkpoint.go:372 msg="writing series" size="205 kB"
level=warn ts=2026-04-29T19:36:53.110646871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:36:54.649733014Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-29T19:36:58.110322118Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:36:58.110391996Z caller=checkpoint.go:372 msg="writing series" size="151 kB"
level=debug ts=2026-04-29T19:37:01.5718707Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-04-29T19:37:03.110791787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:37:03.153934178Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T19:37:05.034111929Z caller=checkpoint.go:372 msg="writing series" size="406 kB"
level=debug ts=2026-04-29T19:37:05.034139483Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=9
ts=2026-04-29T19:37:06.679982661Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:56918"
level=warn ts=2026-04-29T19:37:08.110617639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:37:08.495019469Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-29T19:37:08.894304524Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T19:37:10.597453348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:37:10.606252543Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T19:37:10.696638572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:37:10.703031553Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T19:37:11.957172122Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-04-29T19:37:13.088249397Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:37:13.0887122Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:37:13.110267752Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:37:13.110295932Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:37:13.110306515Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:37:13.110306342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:37:13.724945745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36604"
level=debug ts=2026-04-29T19:37:15.418420327Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
ts=2026-04-29T19:37:15.988356896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=warn ts=2026-04-29T19:37:18.109981471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:37:18.87958033Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-04-29T19:37:22.341152724Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=warn ts=2026-04-29T19:37:23.110589294Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:37:25.803390995Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-04-29T19:37:28.110739681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:37:29.264344064Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=debug ts=2026-04-29T19:37:32.726183687Z caller=checkpoint.go:372 msg="writing series" size="440 kB"
level=warn ts=2026-04-29T19:37:33.110767881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:37:36.18794582Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=warn ts=2026-04-29T19:37:38.110139835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:37:39.659446679Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-04-29T19:37:39.659470383Z caller=checkpoint.go:384 msg="flushing series" totalSize="17 MB" series=10
level=debug ts=2026-04-29T19:37:42.968119263Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T19:37:43.110309419Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T19:37:43.110334198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:37:43.110344932Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:37:43.110356133Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T19:37:43.529349151Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-04-29T19:37:43.5360327Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.425878846s
ts=2026-04-29T19:37:45.992031209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=warn ts=2026-04-29T19:37:48.11028567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:37:49.439500902Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-29T19:37:53.109905489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:37:58.110465519Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:37:59.011645638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:37:59.019019629Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T19:38:03.110174006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:38:08.072458459Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T19:38:08.110339831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:38:09.884047807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:38:09.893349708Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T19:38:13.088078092Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:38:13.088556751Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:38:13.110005845Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-29T19:38:13.110032638Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T19:38:13.110048189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:38:13.110058787Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:38:13.110068954Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T19:38:13.110196722Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=info ts=2026-04-29T19:38:13.110207641Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-29T19:38:13.110225309Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.822µs success=true
level=debug ts=2026-04-29T19:38:13.110396249Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
ts=2026-04-29T19:38:15.995184925Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T19:38:16.572044505Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=warn ts=2026-04-29T19:38:18.110405631Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:38:20.034227974Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-04-29T19:38:23.110514003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:38:23.495947006Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=debug ts=2026-04-29T19:38:26.95739875Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=warn ts=2026-04-29T19:38:28.110753867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:38:30.419069929Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=warn ts=2026-04-29T19:38:33.110257416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:38:33.880546792Z caller=checkpoint.go:372 msg="writing series" size="2.4 kB"
level=debug ts=2026-04-29T19:38:37.341996251Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
level=debug ts=2026-04-29T19:38:38.02784352Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T19:38:38.02959285Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T19:38:38.11025998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:38:40.80458192Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-04-29T19:38:40.804603774Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=9
level=info ts=2026-04-29T19:38:43.110398481Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:38:43.110451837Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-29T19:38:43.110450914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:38:43.110463937Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T19:38:43.398392222Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-29T19:38:43.733516493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42636"
level=debug ts=2026-04-29T19:38:44.26445637Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
ts=2026-04-29T19:38:46.005995563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T19:38:47.726618244Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=warn ts=2026-04-29T19:38:48.110314317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:38:51.188489451Z caller=checkpoint.go:372 msg="writing series" size="404 kB"
level=warn ts=2026-04-29T19:38:53.109845325Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:38:53.864300743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="142 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:38:53.875211525Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T19:38:54.655697366Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-29T19:38:58.110669232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:38:58.110719757Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=info ts=2026-04-29T19:38:58.186998613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:38:58.193496065Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-29T19:38:59.894701613Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:57838"
level=debug ts=2026-04-29T19:39:01.572766741Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-04-29T19:39:03.109853843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:39:05.033991878Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
ts=2026-04-29T19:39:06.698879238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:54352"
level=debug ts=2026-04-29T19:39:07.676328497Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T19:39:08.110469347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:39:08.495228182Z caller=checkpoint.go:372 msg="writing series" size="268 kB"
level=debug ts=2026-04-29T19:39:11.957660176Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=debug ts=2026-04-29T19:39:11.957688789Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=debug ts=2026-04-29T19:39:13.087671895Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:39:13.088744807Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-29T19:39:13.110344956Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:39:13.110377958Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:39:13.110388567Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:39:13.110381916Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:39:14.364070394Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T19:39:15.419099928Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
ts=2026-04-29T19:39:16.008739847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=warn ts=2026-04-29T19:39:18.1104214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:39:18.879905733Z caller=checkpoint.go:372 msg="writing series" size="262 kB"
level=debug ts=2026-04-29T19:39:22.342097664Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=warn ts=2026-04-29T19:39:23.11057648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:39:25.802958231Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=warn ts=2026-04-29T19:39:28.11046988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:39:29.26485059Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-04-29T19:39:32.726958601Z caller=checkpoint.go:372 msg="writing series" size="406 kB"
level=warn ts=2026-04-29T19:39:33.110106618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:39:33.895813048Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:39:36.188361681Z caller=checkpoint.go:372 msg="writing series" size="348 kB"
level=debug ts=2026-04-29T19:39:36.188397179Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-04-29T19:39:36.996414712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="142 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:39:37.005072267Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T19:39:38.110247899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:39:39.650123755Z caller=checkpoint.go:372 msg="writing series" size="223 kB"
level=info ts=2026-04-29T19:39:43.110561843Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:39:43.110590681Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:39:43.11059942Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:39:43.11060147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:39:43.110627682Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
ts=2026-04-29T19:39:43.739228784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:39782"
ts=2026-04-29T19:39:46.015075158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T19:39:46.572211533Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=info ts=2026-04-29T19:39:47.158041312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:39:47.16795493Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:39:47.16797975Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T19:39:47.168005333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:39:47.174451368Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T19:39:48.11013431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:39:50.034280213Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=warn ts=2026-04-29T19:39:53.110489208Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:39:53.495879478Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=debug ts=2026-04-29T19:39:56.957332952Z caller=checkpoint.go:372 msg="writing series" size="410 kB"
level=warn ts=2026-04-29T19:39:58.11075169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:39:59.560241774Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T19:40:00.418341646Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=info ts=2026-04-29T19:40:01.820334718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:40:01.827997403Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T19:40:03.110572243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:40:03.880395429Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=debug ts=2026-04-29T19:40:06.328848363Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:40:07.34171169Z caller=checkpoint.go:372 msg="writing series" size="513 B"
level=warn ts=2026-04-29T19:40:08.110295079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:40:10.803197262Z caller=checkpoint.go:372 msg="writing series" size="598 kB"
level=debug ts=2026-04-29T19:40:10.803222578Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=10
level=info ts=2026-04-29T19:40:13.088222404Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:40:13.089075222Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:40:13.110607127Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:40:13.110633167Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:40:13.11064105Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:40:13.110686724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:40:13.742183714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:44608"
level=debug ts=2026-04-29T19:40:14.265228Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
ts=2026-04-29T19:40:16.018131484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T19:40:17.726613951Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=info ts=2026-04-29T19:40:18.051454248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="165 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:40:18.068177673Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T19:40:18.110363904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:40:21.188422393Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=warn ts=2026-04-29T19:40:23.109994494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:40:24.649786263Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-04-29T19:40:28.110618279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:40:28.110659559Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-04-29T19:40:29.957255577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:40:29.9666469Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T19:40:31.572272387Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-04-29T19:40:33.110479656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:40:35.034026793Z caller=checkpoint.go:372 msg="writing series" size="672 kB"
ts=2026-04-29T19:40:36.7076571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:36456"
level=warn ts=2026-04-29T19:40:38.110315414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:40:38.495517849Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-04-29T19:40:39.80910384Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T19:40:41.88194415Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:40:41.957147613Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=debug ts=2026-04-29T19:40:41.957173672Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-04-29T19:40:43.109978014Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T19:40:43.109989599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:40:43.110007837Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:40:43.11001548Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T19:40:45.419112084Z caller=checkpoint.go:372 msg="writing series" size="367 kB"
ts=2026-04-29T19:40:46.020660367Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=warn ts=2026-04-29T19:40:48.110466708Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:40:48.879726765Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
level=debug ts=2026-04-29T19:40:52.341784111Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=warn ts=2026-04-29T19:40:53.110372788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:40:55.803377218Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-04-29T19:40:58.109863396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:40:59.269201329Z caller=checkpoint.go:372 msg="writing series" size="7.1 MB"
level=debug ts=2026-04-29T19:40:59.269225122Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.8 MB" series=5
ts=2026-04-29T19:40:59.907689147Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:55234"
level=debug ts=2026-04-29T19:41:02.726618688Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=info ts=2026-04-29T19:41:02.813816593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="188 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:41:02.823380595Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T19:41:03.109865605Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:41:06.187849084Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=warn ts=2026-04-29T19:41:08.110501924Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:41:09.64982186Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=info ts=2026-04-29T19:41:13.088066983Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:41:13.088642818Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=debug ts=2026-04-29T19:41:13.09087165Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T19:41:13.110195242Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:41:13.110200096Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:41:13.110225016Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:41:13.110233031Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T19:41:13.111442469Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-04-29T19:41:13.667316876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:41:13.676426398Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-29T19:41:16.023564605Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T19:41:16.572939851Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-04-29T19:41:18.110754024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:41:20.033989155Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-04-29T19:41:23.110213117Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:41:23.49616593Z caller=checkpoint.go:372 msg="writing series" size="921 kB"
level=debug ts=2026-04-29T19:41:23.496186465Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=debug ts=2026-04-29T19:41:26.957463504Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=debug ts=2026-04-29T19:41:27.263764077Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-29T19:41:28.110446022Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:41:29.911398136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:45036"
level=debug ts=2026-04-29T19:41:30.418961454Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-04-29T19:41:33.110244045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:41:33.880416438Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
level=debug ts=2026-04-29T19:41:37.341943768Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=warn ts=2026-04-29T19:41:38.110559865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:41:39.062638711Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:41:40.804069994Z caller=checkpoint.go:372 msg="writing series" size="982 kB"
level=debug ts=2026-04-29T19:41:40.804093881Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-04-29T19:41:43.109797627Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:41:43.109828282Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:41:43.109836188Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:41:43.109851211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:41:44.264920479Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
ts=2026-04-29T19:41:46.026755867Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T19:41:47.726127782Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-04-29T19:41:48.110120853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:41:51.188111244Z caller=checkpoint.go:372 msg="writing series" size="457 kB"
level=warn ts=2026-04-29T19:41:53.11042709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:41:54.652544677Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=info ts=2026-04-29T19:41:56.916459538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:41:56.928776379Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T19:41:58.110558251Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:41:58.110636391Z caller=checkpoint.go:372 msg="writing series" size="178 kB"
level=info ts=2026-04-29T19:41:59.04574814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="188 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:41:59.062140311Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-29T19:41:59.914047292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:52850"
level=debug ts=2026-04-29T19:42:01.572605047Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-04-29T19:42:03.110893799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:42:03.659388402Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T19:42:03.659472993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="607 kB" avg_comp="607 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:42:03.664284342Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T19:42:05.034326726Z caller=checkpoint.go:372 msg="writing series" size="502 kB"
level=debug ts=2026-04-29T19:42:05.034350541Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=debug ts=2026-04-29T19:42:07.919566918Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T19:42:07.921549342Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T19:42:08.11038482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:42:08.495563909Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-29T19:42:11.957075495Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-04-29T19:42:13.088288035Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:42:13.088763399Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:42:13.110368435Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T19:42:13.110373575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:42:13.110403324Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:42:13.110413617Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-29T19:42:13.750063588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:44690"
level=debug ts=2026-04-29T19:42:15.418937982Z caller=checkpoint.go:372 msg="writing series" size="466 kB"
ts=2026-04-29T19:42:16.02978556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=warn ts=2026-04-29T19:42:18.110835454Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:42:18.518828815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:42:18.524103875Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T19:42:18.880162908Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-04-29T19:42:22.341345779Z caller=checkpoint.go:372 msg="writing series" size="257 kB"
level=warn ts=2026-04-29T19:42:23.110865013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:42:25.803125307Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-04-29T19:42:28.110289344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:42:29.265145188Z caller=checkpoint.go:372 msg="writing series" size="212 kB"
level=debug ts=2026-04-29T19:42:32.72620862Z caller=checkpoint.go:372 msg="writing series" size="427 kB"
level=debug ts=2026-04-29T19:42:32.726234953Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=8
level=warn ts=2026-04-29T19:42:33.110546466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:42:33.530134086Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:42:36.1883553Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
ts=2026-04-29T19:42:36.719620737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:53350"
level=warn ts=2026-04-29T19:42:38.1105963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:42:39.274145897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:42:39.284019854Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T19:42:39.677826452Z caller=checkpoint.go:372 msg="writing series" size="27 MB"
level=debug ts=2026-04-29T19:42:39.677852016Z caller=checkpoint.go:384 msg="flushing series" totalSize="28 MB" series=2
level=info ts=2026-04-29T19:42:41.878314958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:42:41.882270199Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T19:42:43.110602262Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:42:43.110630205Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:42:43.110637908Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:42:43.110682618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:42:43.119838161Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-04-29T19:42:43.121668265Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.01131336s
ts=2026-04-29T19:42:43.75323987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:52486"
ts=2026-04-29T19:42:46.032524732Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=warn ts=2026-04-29T19:42:48.110218244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:42:53.110108032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:42:56.963553654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="212 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:42:56.970972803Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:42:56.970994763Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T19:42:56.971022282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="212 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:42:56.976641597Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T19:42:57.958645182Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-29T19:42:58.110253142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:42:59.920950743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:57970"
level=warn ts=2026-04-29T19:43:03.110247641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:43:06.722779385Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:45054"
level=debug ts=2026-04-29T19:43:06.846451219Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T19:43:08.11023105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:43:13.088059494Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:43:13.089210194Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:43:13.109534392Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-29T19:43:13.109758157Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=info ts=2026-04-29T19:43:13.109777522Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-29T19:43:13.109804771Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=21.026µs success=true
level=debug ts=2026-04-29T19:43:13.110022033Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=info ts=2026-04-29T19:43:13.110421811Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:43:13.110463964Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:43:13.110472164Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:43:13.11054031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:43:13.756489315Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:55838"
level=info ts=2026-04-29T19:43:14.203870577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="212 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:43:14.213424864Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-29T19:43:16.035208705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T19:43:16.528428888Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=warn ts=2026-04-29T19:43:18.110249854Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:43:19.946102836Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-04-29T19:43:23.110494249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:43:23.364214238Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-04-29T19:43:26.135653926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:43:26.144301151Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T19:43:26.781303972Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-04-29T19:43:28.110393413Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:43:29.924060497Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:56516"
level=debug ts=2026-04-29T19:43:30.199494887Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-04-29T19:43:33.110847043Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:43:33.616708383Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=info ts=2026-04-29T19:43:35.639569079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:43:35.644288146Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-04-29T19:43:36.726004342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:33252"
level=debug ts=2026-04-29T19:43:37.035157011Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-04-29T19:43:38.11059354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:43:39.250358154Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:43:40.453673081Z caller=checkpoint.go:372 msg="writing series" size="471 kB"
level=debug ts=2026-04-29T19:43:41.009469601Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-29T19:43:43.110538138Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:43:43.110572412Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:43:43.110580665Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:43:43.110586805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:43:43.871064989Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-04-29T19:43:43.87109256Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=10
ts=2026-04-29T19:43:46.037694507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T19:43:47.28746257Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-04-29T19:43:48.110106251Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:43:50.705164744Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=info ts=2026-04-29T19:43:52.377263314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="29 MB" avg_comp="2.9 MB" total_uncomp="236 MB" avg_uncomp="24 MB" full=10 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:43:52.388389414Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T19:43:53.110787425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:43:54.123466713Z caller=checkpoint.go:372 msg="writing series" size="311 kB"
level=debug ts=2026-04-29T19:43:57.540430683Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-29T19:43:58.110280343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:44:00.958580584Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=warn ts=2026-04-29T19:44:03.110300178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:44:04.377374812Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-04-29T19:44:07.794024823Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=warn ts=2026-04-29T19:44:08.110426568Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:44:08.927093421Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:44:11.212185165Z caller=checkpoint.go:372 msg="writing series" size="268 kB"
level=info ts=2026-04-29T19:44:13.088999056Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:44:13.090384253Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:44:13.110765901Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:44:13.110796883Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:44:13.110807676Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:44:13.110838996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:44:13.280567805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:44:13.288426275Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T19:44:14.629707057Z caller=checkpoint.go:372 msg="writing series" size="241 kB"
ts=2026-04-29T19:44:16.040782352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T19:44:18.047188311Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-04-29T19:44:18.110192065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:44:21.469944825Z caller=checkpoint.go:372 msg="writing series" size="532 kB"
level=debug ts=2026-04-29T19:44:21.469973915Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=11
level=warn ts=2026-04-29T19:44:23.110087223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:44:24.88272222Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=warn ts=2026-04-29T19:44:28.110381949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:44:28.299915726Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=info ts=2026-04-29T19:44:31.641405837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:44:31.650992774Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T19:44:31.71789504Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-29T19:44:33.111113356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:44:35.136388919Z caller=checkpoint.go:372 msg="writing series" size="305 kB"
ts=2026-04-29T19:44:36.730562177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:57258"
level=warn ts=2026-04-29T19:44:38.110845161Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:44:38.327775505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="32 MB" avg_comp="2.9 MB" total_uncomp="260 MB" avg_uncomp="24 MB" full=11 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:44:38.335879629Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T19:44:38.556011378Z caller=checkpoint.go:372 msg="writing series" size="273 kB"
level=debug ts=2026-04-29T19:44:41.971637332Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=debug ts=2026-04-29T19:44:42.042580072Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-29T19:44:43.110725133Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:44:43.110759024Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:44:43.110766297Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:44:43.11083715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:44:43.169615236Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:44:45.388752079Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-04-29T19:44:46.043629126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=warn ts=2026-04-29T19:44:48.109891618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:44:48.806615791Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=debug ts=2026-04-29T19:44:48.806640036Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=debug ts=2026-04-29T19:44:52.224804085Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-04-29T19:44:53.110812822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:44:55.641727843Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=warn ts=2026-04-29T19:44:58.110942317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:44:59.05981696Z caller=checkpoint.go:372 msg="writing series" size="494 kB"
ts=2026-04-29T19:44:59.932200707Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:60696"
level=debug ts=2026-04-29T19:45:02.477483397Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-04-29T19:45:03.110793261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:45:05.895582478Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=warn ts=2026-04-29T19:45:08.110666169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:45:09.313222373Z caller=checkpoint.go:372 msg="writing series" size="513 B"
level=debug ts=2026-04-29T19:45:11.996720488Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T19:45:12.222371897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:45:12.23117423Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T19:45:12.731029634Z caller=checkpoint.go:372 msg="writing series" size="688 kB"
level=debug ts=2026-04-29T19:45:12.731050699Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=7
level=debug ts=2026-04-29T19:45:13.087937681Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:45:13.087945534Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:45:13.089216039Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-29T19:45:13.089257375Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974990 n=1
level=debug ts=2026-04-29T19:45:13.08929672Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974990 n=1
level=info ts=2026-04-29T19:45:13.110388614Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:45:13.110407109Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:45:13.110415034Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:45:13.1104472Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:45:16.046921369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T19:45:16.148786Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=warn ts=2026-04-29T19:45:18.110426847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:45:19.566316094Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=info ts=2026-04-29T19:45:22.691882424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:45:22.696833748Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T19:45:22.983688603Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
level=warn ts=2026-04-29T19:45:23.110338605Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:45:26.402090515Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=info ts=2026-04-29T19:45:26.942571014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="32 MB" avg_comp="2.9 MB" total_uncomp="260 MB" avg_uncomp="24 MB" full=11 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:45:26.94973432Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T19:45:28.110301241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:45:29.820064362Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-04-29T19:45:29.820085921Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=5
ts=2026-04-29T19:45:29.935767878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:44404"
level=warn ts=2026-04-29T19:45:33.110426406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:45:33.236805961Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=debug ts=2026-04-29T19:45:36.656115022Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-29T19:45:36.656137386Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=warn ts=2026-04-29T19:45:38.110233849Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:45:38.251715482Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T19:45:40.072151839Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-04-29T19:45:43.110382059Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:45:43.110413894Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:45:43.110423773Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:45:43.11041203Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:45:43.128785364Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:45:43.490514807Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
ts=2026-04-29T19:45:46.049383106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T19:45:46.908619621Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
level=warn ts=2026-04-29T19:45:48.110711769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:45:50.326545686Z caller=checkpoint.go:372 msg="writing series" size="177 kB"
level=warn ts=2026-04-29T19:45:53.11042778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:45:53.743201901Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=debug ts=2026-04-29T19:45:57.161679985Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=warn ts=2026-04-29T19:45:58.110397745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:45:59.939357389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:43564"
level=debug ts=2026-04-29T19:46:00.587556779Z caller=checkpoint.go:372 msg="writing series" size="7.7 MB"
level=debug ts=2026-04-29T19:46:00.587581725Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.5 MB" series=7
level=warn ts=2026-04-29T19:46:03.110489192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:46:03.379558566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:46:03.391114652Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:46:03.391142561Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T19:46:03.391170145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:46:03.398145966Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T19:46:03.997038119Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
ts=2026-04-29T19:46:06.737790104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:52442"
level=debug ts=2026-04-29T19:46:07.414691789Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=info ts=2026-04-29T19:46:07.511080311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:46:07.515485533Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-29T19:46:08.10990387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:46:10.058684887Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:46:10.832577784Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-04-29T19:46:13.08792569Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:46:13.088357025Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=warn ts=2026-04-29T19:46:13.110508451Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:46:13.110550465Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:46:13.110563813Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:46:13.110573183Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T19:46:13.816824527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:46:13.825318391Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T19:46:14.249733182Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
ts=2026-04-29T19:46:16.052021965Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T19:46:17.667354921Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=warn ts=2026-04-29T19:46:18.110970648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:46:21.085757465Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-04-29T19:46:21.390326433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="32 MB" avg_comp="2.9 MB" total_uncomp="260 MB" avg_uncomp="24 MB" full=11 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:46:21.398047556Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T19:46:22.995243Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T19:46:22.996838302Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T19:46:23.110481096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:46:24.50318641Z caller=checkpoint.go:372 msg="writing series" size="886 kB"
level=debug ts=2026-04-29T19:46:24.503209465Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=debug ts=2026-04-29T19:46:27.920791776Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=warn ts=2026-04-29T19:46:28.110489575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:46:31.338064412Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-04-29T19:46:33.110362119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:46:34.755866138Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
level=debug ts=2026-04-29T19:46:38.079562946Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T19:46:38.110802412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:46:38.174040786Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=debug ts=2026-04-29T19:46:41.592549979Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-04-29T19:46:41.592572799Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=5
level=info ts=2026-04-29T19:46:43.10978179Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:46:43.109814899Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:46:43.109823483Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:46:43.109863436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:46:45.009013794Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-04-29T19:46:46.055170348Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=warn ts=2026-04-29T19:46:48.110065825Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:46:48.42707054Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-04-29T19:46:49.617979164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:46:49.630739562Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T19:46:51.844582712Z caller=checkpoint.go:372 msg="writing series" size="776 kB"
level=warn ts=2026-04-29T19:46:53.110312267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:46:55.262390132Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-04-29T19:46:58.110090426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:46:58.680769746Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
level=debug ts=2026-04-29T19:47:00.977204621Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T19:47:02.097641131Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=debug ts=2026-04-29T19:47:02.097663603Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-04-29T19:47:03.110758305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:47:05.515482181Z caller=checkpoint.go:372 msg="writing series" size="484 kB"
level=info ts=2026-04-29T19:47:05.917487629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:47:05.923168642Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-04-29T19:47:06.742902961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:37452"
level=warn ts=2026-04-29T19:47:08.110390336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:47:08.436849332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="35 MB" avg_comp="2.9 MB" total_uncomp="284 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:47:08.443662287Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T19:47:08.93365297Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-29T19:47:11.201307715Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:47:12.350961193Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-04-29T19:47:13.087897863Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:47:13.088418583Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:47:13.109794594Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:47:13.109824741Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:47:13.10983381Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:47:13.109880518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:47:15.768652273Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-04-29T19:47:15.768680923Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
ts=2026-04-29T19:47:16.058528671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=warn ts=2026-04-29T19:47:18.109925072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:47:19.186378243Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-04-29T19:47:22.604065978Z caller=checkpoint.go:372 msg="writing series" size="347 kB"
level=warn ts=2026-04-29T19:47:23.109832379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:47:26.022034245Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-04-29T19:47:28.110703529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:47:29.4397786Z caller=checkpoint.go:372 msg="writing series" size="627 kB"
ts=2026-04-29T19:47:29.948015455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:43886"
level=debug ts=2026-04-29T19:47:32.857531734Z caller=checkpoint.go:372 msg="writing series" size="562 kB"
level=debug ts=2026-04-29T19:47:32.857553993Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=warn ts=2026-04-29T19:47:33.110494717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:47:36.275309042Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=warn ts=2026-04-29T19:47:38.110360147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:47:39.732109502Z caller=checkpoint.go:372 msg="writing series" size="38 MB"
level=debug ts=2026-04-29T19:47:39.732141223Z caller=checkpoint.go:384 msg="flushing series" totalSize="38 MB" series=2
level=info ts=2026-04-29T19:47:43.110474399Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:47:43.1105147Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:47:43.110525666Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:47:43.110550434Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:47:43.112620554Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-04-29T19:47:43.114489474Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.004524779s
level=debug ts=2026-04-29T19:47:45.805919585Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:47:45.937270966Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-29T19:47:45.991552582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:47:45.998498781Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-29T19:47:46.061599273Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=warn ts=2026-04-29T19:47:48.110133311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:47:50.242629511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="38 MB" avg_comp="2.9 MB" total_uncomp="308 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:47:50.251990579Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T19:47:53.110780848Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:47:58.110487997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:47:59.786488592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:47:59.79037486Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-29T19:48:03.110662078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:48:06.747936648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:59430"
level=warn ts=2026-04-29T19:48:08.110650743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:48:11.06677132Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T19:48:13.088560826Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:48:13.089380397Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:48:13.109682588Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-29T19:48:13.110061885Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=info ts=2026-04-29T19:48:13.110076417Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-29T19:48:13.110096004Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.766µs success=true
level=warn ts=2026-04-29T19:48:13.110149741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:48:13.110165478Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:48:13.110185661Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:48:13.110193378Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T19:48:13.110304488Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
ts=2026-04-29T19:48:13.775311481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:43678"
ts=2026-04-29T19:48:16.063915287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T19:48:16.485947563Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=warn ts=2026-04-29T19:48:18.110640564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:48:19.861106689Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-04-29T19:48:23.110489031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:48:23.235871674Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-04-29T19:48:26.611123427Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-04-29T19:48:28.110705549Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:48:29.985939882Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-04-29T19:48:31.807255229Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-29T19:48:32.888494893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:48:32.896504443Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T19:48:33.110202048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:48:33.360729737Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=debug ts=2026-04-29T19:48:36.736084468Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
ts=2026-04-29T19:48:36.750261305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:57250"
level=debug ts=2026-04-29T19:48:37.877788758Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T19:48:38.110219512Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:48:40.111163625Z caller=checkpoint.go:372 msg="writing series" size="418 kB"
level=info ts=2026-04-29T19:48:43.11053172Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:48:43.110560244Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-29T19:48:43.110549296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:48:43.110568451Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T19:48:43.489095089Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-04-29T19:48:43.489118907Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.6 MB" series=10
ts=2026-04-29T19:48:43.777846112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36542"
ts=2026-04-29T19:48:46.066784249Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T19:48:46.860305571Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=info ts=2026-04-29T19:48:47.147463152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="38 MB" avg_comp="2.9 MB" total_uncomp="308 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:48:47.157483718Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T19:48:48.11041222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:48:50.236315922Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=warn ts=2026-04-29T19:48:53.110836431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:48:53.611278252Z caller=checkpoint.go:372 msg="writing series" size="577 kB"
level=info ts=2026-04-29T19:48:56.257756582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:48:56.262225311Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:48:56.262250049Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T19:48:56.262273357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:48:56.266051171Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T19:48:56.986223031Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-29T19:48:58.110323469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:49:00.360734803Z caller=checkpoint.go:372 msg="writing series" size="8.2 kB"
level=warn ts=2026-04-29T19:49:03.110731581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:49:03.735374787Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-04-29T19:49:07.111020845Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=warn ts=2026-04-29T19:49:08.110560693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:49:10.486309494Z caller=checkpoint.go:372 msg="writing series" size="268 kB"
level=debug ts=2026-04-29T19:49:10.486335609Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=debug ts=2026-04-29T19:49:10.799165361Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T19:49:12.222359691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:49:12.229052167Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T19:49:13.087908077Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:49:13.087903432Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:49:13.110769811Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:49:13.110801014Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:49:13.110808277Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:49:13.110817395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:49:13.78098554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:60330"
level=debug ts=2026-04-29T19:49:13.860649694Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=info ts=2026-04-29T19:49:13.960022653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:49:13.967518975Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-29T19:49:16.069278433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T19:49:17.23627135Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-04-29T19:49:18.110337997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:49:20.611241289Z caller=checkpoint.go:372 msg="writing series" size="532 kB"
level=warn ts=2026-04-29T19:49:23.110701902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:49:23.985397152Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=debug ts=2026-04-29T19:49:27.362550961Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=warn ts=2026-04-29T19:49:28.110426353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:49:29.959034721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:51386"
level=debug ts=2026-04-29T19:49:30.736270148Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-29T19:49:33.110104739Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:49:34.110892433Z caller=checkpoint.go:372 msg="writing series" size="429 kB"
level=debug ts=2026-04-29T19:49:34.110915209Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=debug ts=2026-04-29T19:49:34.317153461Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-29T19:49:37.381028743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="332 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:49:37.387958393Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:49:37.387978355Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T19:49:37.388011185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="332 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:49:37.39487979Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T19:49:37.486416224Z caller=checkpoint.go:372 msg="writing series" size="388 kB"
level=warn ts=2026-04-29T19:49:38.110175912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:49:38.740871416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:49:38.744739178Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T19:49:40.86092624Z caller=checkpoint.go:372 msg="writing series" size="234 kB"
level=info ts=2026-04-29T19:49:43.110643562Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:49:43.110672894Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:49:43.110681347Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:49:43.110684395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:49:44.236269873Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
ts=2026-04-29T19:49:46.072813755Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T19:49:46.303498951Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:49:47.611239233Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
level=warn ts=2026-04-29T19:49:48.110664129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:49:50.985716607Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
level=debug ts=2026-04-29T19:49:50.985741271Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-04-29T19:49:53.109843255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:49:54.360768011Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=info ts=2026-04-29T19:49:56.335855369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="332 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:49:56.346915154Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T19:49:57.736271524Z caller=checkpoint.go:372 msg="writing series" size="619 kB"
level=warn ts=2026-04-29T19:49:58.110222647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:49:59.962241289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:47410"
level=debug ts=2026-04-29T19:50:01.110922868Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=info ts=2026-04-29T19:50:02.969017397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:50:02.977159187Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T19:50:03.110023778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:50:04.486356159Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
ts=2026-04-29T19:50:06.759569241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:40526"
level=debug ts=2026-04-29T19:50:07.86129136Z caller=checkpoint.go:372 msg="writing series" size="513 B"
level=warn ts=2026-04-29T19:50:08.110184846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:50:11.236187295Z caller=checkpoint.go:372 msg="writing series" size="669 kB"
level=debug ts=2026-04-29T19:50:11.236212902Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=debug ts=2026-04-29T19:50:12.334351538Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T19:50:13.087948047Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:50:13.087948317Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:50:13.109779988Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:50:13.109809601Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:50:13.109817562Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:50:13.109911045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:50:13.785936265Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:53156"
level=debug ts=2026-04-29T19:50:14.610466523Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
ts=2026-04-29T19:50:16.075731701Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=info ts=2026-04-29T19:50:17.887833147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="332 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:50:17.898571673Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T19:50:17.985785531Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-04-29T19:50:18.110875207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:50:21.361336583Z caller=checkpoint.go:372 msg="writing series" size="284 kB"
level=warn ts=2026-04-29T19:50:23.110418024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:50:24.735385062Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=warn ts=2026-04-29T19:50:28.110456972Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:50:28.112539118Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-04-29T19:50:28.112554676Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=5
ts=2026-04-29T19:50:29.965383394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:49012"
level=info ts=2026-04-29T19:50:31.112838582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:50:31.11773894Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T19:50:31.485856703Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=debug ts=2026-04-29T19:50:32.031964297Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T19:50:32.033513782Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T19:50:33.110687453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:50:34.863503482Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-04-29T19:50:34.863559594Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=2
level=warn ts=2026-04-29T19:50:38.110506044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:50:38.235761176Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-04-29T19:50:38.986666261Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:50:41.6108055Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=info ts=2026-04-29T19:50:43.110480743Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:50:43.110511542Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:50:43.110519266Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:50:43.110536251Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:50:43.79395532Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:47064"
level=debug ts=2026-04-29T19:50:44.986498512Z caller=checkpoint.go:372 msg="writing series" size="356 kB"
ts=2026-04-29T19:50:46.078559056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=warn ts=2026-04-29T19:50:48.110867081Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:50:48.360749499Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=debug ts=2026-04-29T19:50:51.736014995Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=warn ts=2026-04-29T19:50:53.110401743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:50:55.111310704Z caller=checkpoint.go:372 msg="writing series" size="171 kB"
level=warn ts=2026-04-29T19:50:58.110166188Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:50:58.494950508Z caller=checkpoint.go:372 msg="writing series" size="8.3 MB"
level=debug ts=2026-04-29T19:50:58.494974152Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.4 MB" series=7
level=info ts=2026-04-29T19:51:00.110456294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:51:00.121021941Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T19:51:01.861280629Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=warn ts=2026-04-29T19:51:03.110097133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:51:05.236005047Z caller=checkpoint.go:372 msg="writing series" size="935 B"
ts=2026-04-29T19:51:06.764053981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:60362"
level=debug ts=2026-04-29T19:51:07.771562873Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T19:51:07.771612509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T19:51:07.780416598Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-zciz\"}"
level=warn ts=2026-04-29T19:51:08.110511362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:51:08.611317188Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-04-29T19:51:09.292575154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="332 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:51:09.301402893Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T19:51:11.986009546Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=info ts=2026-04-29T19:51:13.087998472Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:51:13.088603695Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:51:13.109996289Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:51:13.110024555Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:51:13.110032072Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:51:13.110048999Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:51:15.360662517Z caller=checkpoint.go:372 msg="writing series" size="77 kB"
ts=2026-04-29T19:51:16.081758339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=warn ts=2026-04-29T19:51:18.110648559Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:51:18.736085039Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-04-29T19:51:22.111130207Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-29T19:51:22.111153236Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=7
level=warn ts=2026-04-29T19:51:23.110349931Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:51:24.819259154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T19:51:24.828800577Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T19:51:25.486454874Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=warn ts=2026-04-29T19:51:28.109857939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:51:28.861256088Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
level=info ts=2026-04-29T19:51:30.546791378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:51:30.558613327Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T19:51:32.235679555Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-04-29T19:51:33.109833292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:51:33.796477661Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T19:51:35.611046816Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
level=debug ts=2026-04-29T19:51:35.746494679Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T19:51:38.110409404Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:51:38.985975171Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=debug ts=2026-04-29T19:51:42.361363496Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-04-29T19:51:42.361385045Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=6
level=info ts=2026-04-29T19:51:43.110562364Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:51:43.110589949Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:51:43.110598914Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:51:43.1105857Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:51:45.735349852Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-04-29T19:51:46.08446133Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=warn ts=2026-04-29T19:51:48.110728926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:51:49.110620885Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-04-29T19:51:52.486578123Z caller=checkpoint.go:372 msg="writing series" size="932 kB"
level=warn ts=2026-04-29T19:51:53.110063955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:51:55.861307266Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-04-29T19:51:58.110727027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:51:59.067692571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T19:51:59.078307732Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T19:51:59.235908273Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=debug ts=2026-04-29T19:51:59.235932264Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-04-29T19:51:59.388481308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:51:59.395824734Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T19:52:02.610845828Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=warn ts=2026-04-29T19:52:03.110794094Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:52:05.986717768Z caller=checkpoint.go:372 msg="writing series" size="768 kB"
level=info ts=2026-04-29T19:52:07.97488646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="356 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:52:07.983569482Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T19:52:08.110967196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:52:08.481490718Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T19:52:09.361058801Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-29T19:52:10.119339494Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:52:12.735771656Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=debug ts=2026-04-29T19:52:13.087635134Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:52:13.088674849Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-29T19:52:13.110211262Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:52:13.11023538Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:52:13.110243469Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:52:13.110241578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:52:16.087673854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T19:52:16.110967814Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-04-29T19:52:16.110992319Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=warn ts=2026-04-29T19:52:18.110152548Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:52:19.485500075Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-04-29T19:52:20.879354271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:52:20.884743117Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T19:52:22.861487351Z caller=checkpoint.go:372 msg="writing series" size="347 kB"
level=warn ts=2026-04-29T19:52:23.110425927Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:52:26.23613093Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-04-29T19:52:28.110319477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:52:29.610815957Z caller=checkpoint.go:372 msg="writing series" size="627 kB"
ts=2026-04-29T19:52:29.975550698Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:57904"
level=debug ts=2026-04-29T19:52:32.986352377Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-04-29T19:52:32.986375109Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=warn ts=2026-04-29T19:52:33.110372344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:52:36.361144461Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=warn ts=2026-04-29T19:52:38.110308393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:52:39.094619261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T19:52:39.103703004Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T19:52:39.774611975Z caller=checkpoint.go:372 msg="writing series" size="45 MB"
level=debug ts=2026-04-29T19:52:39.774652141Z caller=checkpoint.go:384 msg="flushing series" totalSize="45 MB" series=2
level=info ts=2026-04-29T19:52:42.726721466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:52:42.733524368Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:52:42.733551552Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T19:52:42.733577365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:52:42.740456504Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T19:52:43.109932186Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:52:43.109963448Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:52:43.109973586Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:52:43.110009176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:52:43.121889299Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-04-29T19:52:43.12378553Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.013541597s
level=debug ts=2026-04-29T19:52:43.525026488Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:52:45.975236924Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T19:52:46.090815847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=warn ts=2026-04-29T19:52:48.110199688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:52:53.110144667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:52:56.193548865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="356 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:52:56.20305276Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T19:52:58.11053963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:53:02.314626513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:53:02.322412239Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T19:53:03.110343954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:53:08.110076365Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:53:11.795637663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:53:11.801805435Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T19:53:12.902156121Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T19:53:12.90219195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T19:53:12.909320991Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-29T19:53:13.088048647Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:53:13.088517474Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=warn ts=2026-04-29T19:53:13.109948253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:53:13.112485279Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-29T19:53:13.112614972Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:53:13.112636642Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:53:13.112647172Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T19:53:13.11268212Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=info ts=2026-04-29T19:53:13.112703807Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-29T19:53:13.112728853Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=17.806µs success=true
level=debug ts=2026-04-29T19:53:13.112963417Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
ts=2026-04-29T19:53:16.094190205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T19:53:16.489093158Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=warn ts=2026-04-29T19:53:18.10989246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:53:19.863321221Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=info ts=2026-04-29T19:53:21.719332319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T19:53:21.728419201Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-zciz\"}"
level=warn ts=2026-04-29T19:53:23.110415011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:53:23.238848344Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-04-29T19:53:23.949491053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T19:53:23.957192367Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T19:53:26.613714723Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=warn ts=2026-04-29T19:53:28.110697682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:53:29.980516261Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:54282"
level=debug ts=2026-04-29T19:53:29.988378483Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-04-29T19:53:32.650255648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:53:32.657938184Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T19:53:33.110637263Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:53:33.363179392Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=debug ts=2026-04-29T19:53:36.737986494Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-04-29T19:53:37.999869424Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-29T19:53:38.110222555Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:53:40.114362173Z caller=checkpoint.go:372 msg="writing series" size="591 kB"
level=debug ts=2026-04-29T19:53:40.114383247Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=info ts=2026-04-29T19:53:43.110326555Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:53:43.11035651Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:53:43.110367268Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:53:43.110361896Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:53:43.491993696Z caller=checkpoint.go:372 msg="writing series" size="3.1 MB"
level=debug ts=2026-04-29T19:53:43.492017459Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=1
ts=2026-04-29T19:53:46.097056388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T19:53:46.863037093Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-04-29T19:53:48.110188085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:53:48.124376938Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:53:50.238848597Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=warn ts=2026-04-29T19:53:53.1108881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:53:53.613291902Z caller=checkpoint.go:372 msg="writing series" size="370 kB"
level=info ts=2026-04-29T19:53:53.815663045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="45 MB" avg_comp="2.8 MB" total_uncomp="368 MB" avg_uncomp="23 MB" full=15 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:53:53.826269359Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T19:53:54.379060603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T19:53:54.387026651Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T19:53:56.988479388Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-29T19:53:58.110473692Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:54:00.36346729Z caller=checkpoint.go:372 msg="writing series" size="9.3 kB"
level=info ts=2026-04-29T19:54:02.458243589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:54:02.462884058Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-29T19:54:03.110136094Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:54:03.738867353Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
ts=2026-04-29T19:54:06.778053346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:44014"
level=debug ts=2026-04-29T19:54:07.113751199Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=warn ts=2026-04-29T19:54:08.11042767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:54:10.488700898Z caller=checkpoint.go:372 msg="writing series" size="268 kB"
level=info ts=2026-04-29T19:54:12.165178406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T19:54:12.174577667Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T19:54:13.087816562Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:54:13.088865605Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-29T19:54:13.110388082Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:54:13.110419277Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:54:13.110429201Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:54:13.110462374Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:54:13.818417404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:57280"
level=debug ts=2026-04-29T19:54:13.863922858Z caller=checkpoint.go:372 msg="writing series" size="151 kB"
ts=2026-04-29T19:54:16.099808252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T19:54:17.238588209Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-04-29T19:54:18.110465016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:54:18.416817191Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:54:20.614105974Z caller=checkpoint.go:372 msg="writing series" size="532 kB"
level=debug ts=2026-04-29T19:54:20.614127812Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=11
level=info ts=2026-04-29T19:54:20.90241675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="74 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:54:20.909826028Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T19:54:22.442803881Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T19:54:22.444551131Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T19:54:23.109921753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:54:23.989007297Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-04-29T19:54:27.363725114Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=warn ts=2026-04-29T19:54:28.110270953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:54:30.738831985Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-29T19:54:33.110337416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:54:34.113676904Z caller=checkpoint.go:372 msg="writing series" size="357 kB"
level=info ts=2026-04-29T19:54:36.488858934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="45 MB" avg_comp="2.8 MB" total_uncomp="368 MB" avg_uncomp="23 MB" full=15 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:54:36.50156247Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-29T19:54:36.78111527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:55844"
level=debug ts=2026-04-29T19:54:37.488872525Z caller=checkpoint.go:372 msg="writing series" size="310 kB"
level=warn ts=2026-04-29T19:54:38.110292398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:54:40.863362935Z caller=checkpoint.go:372 msg="writing series" size="243 kB"
level=info ts=2026-04-29T19:54:43.110724226Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:54:43.110755819Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:54:43.110763873Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:54:43.110774703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:54:43.821452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:40714"
level=debug ts=2026-04-29T19:54:44.238159604Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
level=debug ts=2026-04-29T19:54:44.238182641Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=debug ts=2026-04-29T19:54:44.302567887Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-29T19:54:46.103254092Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T19:54:47.614461942Z caller=checkpoint.go:372 msg="writing series" size="761 kB"
level=warn ts=2026-04-29T19:54:48.110466302Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:54:50.988826303Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
level=info ts=2026-04-29T19:54:52.324636355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T19:54:52.334068159Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-29T19:54:53.110484345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:54:54.363839486Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-04-29T19:54:57.738874123Z caller=checkpoint.go:372 msg="writing series" size="745 kB"
level=debug ts=2026-04-29T19:54:57.738895825Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=4
level=warn ts=2026-04-29T19:54:58.11057218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:54:59.807140262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:54:59.811196699Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T19:55:01.113737436Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=warn ts=2026-04-29T19:55:03.109859028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:55:04.48832261Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=debug ts=2026-04-29T19:55:07.863420831Z caller=checkpoint.go:372 msg="writing series" size="555 B"
level=warn ts=2026-04-29T19:55:08.110115269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:55:11.238622713Z caller=checkpoint.go:372 msg="writing series" size="621 kB"
level=info ts=2026-04-29T19:55:11.877814359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T19:55:11.887347385Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T19:55:13.088273686Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:55:13.090806901Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:55:13.11016723Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:55:13.110188192Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:55:13.110196265Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:55:13.11020415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:55:14.61397951Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-04-29T19:55:14.613996013Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
ts=2026-04-29T19:55:16.106772272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=info ts=2026-04-29T19:55:16.986228207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="74 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:55:16.992666131Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T19:55:17.152429874Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T19:55:17.988896494Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-04-29T19:55:18.110325832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:55:18.638516484Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:55:21.363952627Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=info ts=2026-04-29T19:55:22.275630178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="45 MB" avg_comp="2.8 MB" total_uncomp="368 MB" avg_uncomp="23 MB" full=15 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:55:22.282481809Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T19:55:23.11047123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:55:24.738523548Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=warn ts=2026-04-29T19:55:28.110602728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:55:28.116257503Z caller=checkpoint.go:372 msg="writing series" size="4.6 MB"
level=debug ts=2026-04-29T19:55:28.116272474Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.8 MB" series=4
ts=2026-04-29T19:55:29.994085223Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:35134"
level=debug ts=2026-04-29T19:55:31.488533732Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-04-29T19:55:33.110661573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:55:34.864327707Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-04-29T19:55:34.864350986Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=2
level=warn ts=2026-04-29T19:55:38.110858973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:55:38.238254806Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-04-29T19:55:41.613409749Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=info ts=2026-04-29T19:55:43.10979471Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:55:43.109832795Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:55:43.10984191Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:55:43.109871392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:55:44.988312992Z caller=checkpoint.go:372 msg="writing series" size="495 kB"
ts=2026-04-29T19:55:46.109825058Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=warn ts=2026-04-29T19:55:48.10990137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:55:48.363387196Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=info ts=2026-04-29T19:55:51.461493335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T19:55:51.47410519Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T19:55:51.739114346Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=warn ts=2026-04-29T19:55:53.109914313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:55:53.91792053Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T19:55:54.371966438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:55:54.376593692Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:55:54.376616441Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T19:55:54.376637227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:55:54.38020129Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T19:55:55.113659822Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=debug ts=2026-04-29T19:55:55.113681211Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-04-29T19:55:58.110815817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:55:58.501127037Z caller=checkpoint.go:372 msg="writing series" size="8.7 MB"
level=debug ts=2026-04-29T19:55:58.501150887Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.7 MB" series=1
level=info ts=2026-04-29T19:56:00.464864076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T19:56:00.472077042Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T19:56:01.863146268Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=warn ts=2026-04-29T19:56:03.110736831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:56:03.431914743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="74 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:56:03.438894788Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T19:56:05.238500017Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=warn ts=2026-04-29T19:56:08.110140543Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:56:08.613522236Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-04-29T19:56:10.908241247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:56:10.91209276Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T19:56:11.988711366Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-04-29T19:56:12.688734034Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-29T19:56:13.026826422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="392 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:56:13.034732208Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:56:13.034755377Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T19:56:13.034791579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="392 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:56:13.041172828Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T19:56:13.088029342Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:56:13.088478692Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:56:13.109921645Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T19:56:13.109936795Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:56:13.1099504Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:56:13.109961953Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T19:56:15.363209731Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
ts=2026-04-29T19:56:16.11267011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=warn ts=2026-04-29T19:56:18.110318666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:56:18.738501564Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-04-29T19:56:22.114635915Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-04-29T19:56:22.114656601Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=7
level=warn ts=2026-04-29T19:56:23.110529645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:56:24.19456307Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:56:25.48883881Z caller=checkpoint.go:372 msg="writing series" size="204 kB"
level=info ts=2026-04-29T19:56:27.68493068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="392 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:56:27.695544998Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T19:56:28.110386529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:56:28.863514736Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
ts=2026-04-29T19:56:29.999979446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:48850"
level=info ts=2026-04-29T19:56:32.025492354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:56:32.029840028Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T19:56:32.238599277Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-04-29T19:56:33.110177764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:56:35.613218064Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
ts=2026-04-29T19:56:36.802269466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:59036"
level=warn ts=2026-04-29T19:56:38.11068627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:56:38.988857779Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=info ts=2026-04-29T19:56:42.197647822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T19:56:42.208120034Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T19:56:42.365809899Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-04-29T19:56:42.365831439Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=6
level=info ts=2026-04-29T19:56:43.1107051Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T19:56:43.110717855Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:56:43.110741864Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:56:43.110752619Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T19:56:43.756456335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T19:56:43.764277422Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-zciz\"}"
ts=2026-04-29T19:56:43.830082011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:60064"
level=debug ts=2026-04-29T19:56:45.738559466Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-04-29T19:56:46.116041225Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=warn ts=2026-04-29T19:56:48.110588318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:56:49.113876814Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-04-29T19:56:51.323047091Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:56:51.363983212Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T19:56:52.489036435Z caller=checkpoint.go:372 msg="writing series" size="846 kB"
level=warn ts=2026-04-29T19:56:53.110662377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:56:55.863221476Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-04-29T19:56:57.034733399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="392 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:56:57.042090676Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T19:56:58.110083542Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:56:58.956548812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="74 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:56:58.964118137Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T19:56:59.238110218Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=debug ts=2026-04-29T19:56:59.238134005Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=debug ts=2026-04-29T19:57:02.61368075Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-04-29T19:57:03.110265991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:57:05.988655941Z caller=checkpoint.go:372 msg="writing series" size="776 kB"
ts=2026-04-29T19:57:06.805785849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:41662"
level=warn ts=2026-04-29T19:57:08.110565811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:57:09.363907389Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-29T19:57:12.738727334Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-04-29T19:57:13.088919052Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:57:13.089886544Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:57:13.110311586Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:57:13.110329889Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:57:13.1103381Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:57:13.110334286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:57:16.113828769Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-04-29T19:57:16.113850093Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
ts=2026-04-29T19:57:16.11870698Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=warn ts=2026-04-29T19:57:18.110470729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:57:19.488962123Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-04-29T19:57:20.785708902Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:57:22.863668059Z caller=checkpoint.go:372 msg="writing series" size="347 kB"
level=warn ts=2026-04-29T19:57:23.110755658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:57:26.238824969Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-04-29T19:57:27.311636856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:57:27.318158132Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-29T19:57:28.110616246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:57:29.614244332Z caller=checkpoint.go:372 msg="writing series" size="627 kB"
level=debug ts=2026-04-29T19:57:32.357784572Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-29T19:57:32.435067172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T19:57:32.44155882Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T19:57:32.988601684Z caller=checkpoint.go:372 msg="writing series" size="603 kB"
level=debug ts=2026-04-29T19:57:32.988626555Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=warn ts=2026-04-29T19:57:33.109961826Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:57:36.364097936Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=warn ts=2026-04-29T19:57:38.110732714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:57:39.160404648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="74 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:57:39.167258646Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T19:57:39.777617656Z caller=checkpoint.go:372 msg="writing series" size="50 MB"
level=debug ts=2026-04-29T19:57:39.777642417Z caller=checkpoint.go:384 msg="flushing series" totalSize="50 MB" series=2
level=info ts=2026-04-29T19:57:41.761031496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T19:57:41.769026574Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-zciz\"}"
level=error ts=2026-04-29T19:57:41.769049782Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T19:57:41.769072332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T19:57:41.776517749Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T19:57:43.109895782Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:57:43.10992486Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:57:43.109933164Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:57:43.109963526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:57:43.125036137Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-04-29T19:57:43.126790735Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.013886906s
ts=2026-04-29T19:57:43.834915904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:46146"
ts=2026-04-29T19:57:46.121816468Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T19:57:47.18039109Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T19:57:48.110181687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:57:48.967812911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="48 MB" avg_comp="2.8 MB" total_uncomp="392 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:57:48.977388653Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T19:57:51.981924064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T19:57:51.991827777Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-zciz\"}"
level=warn ts=2026-04-29T19:57:53.109964658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T19:57:58.110059861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:58:00.007807979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:56876"
level=warn ts=2026-04-29T19:58:03.110492694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:58:06.810023737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:37092"
level=warn ts=2026-04-29T19:58:08.110241188Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:58:11.78977681Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T19:58:13.088010117Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:58:13.088554554Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:58:13.109971397Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-29T19:58:13.109999428Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T19:58:13.110008592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:58:13.11002034Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:58:13.110028906Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T19:58:13.110110641Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=info ts=2026-04-29T19:58:13.110119081Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-29T19:58:13.110136584Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.117µs success=true
level=debug ts=2026-04-29T19:58:13.110352689Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=info ts=2026-04-29T19:58:14.200337602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T19:58:14.209635781Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-29T19:58:16.124152652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=info ts=2026-04-29T19:58:16.265606181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:58:16.26936746Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T19:58:16.48657556Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=warn ts=2026-04-29T19:58:18.109889324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:58:18.66542955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T19:58:18.667089285Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-29T19:58:19.861142631Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-04-29T19:58:23.110535478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:58:23.235899972Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-04-29T19:58:25.954988686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="74 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:58:25.9618494Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T19:58:26.610738638Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=info ts=2026-04-29T19:58:28.040811862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T19:58:28.047445185Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-zciz\"}"
level=warn ts=2026-04-29T19:58:28.110774425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:58:29.985777289Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-04-29T19:58:33.110167248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:58:33.361059998Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=debug ts=2026-04-29T19:58:36.735830983Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=warn ts=2026-04-29T19:58:38.110549165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:58:40.111580699Z caller=checkpoint.go:372 msg="writing series" size="570 kB"
level=debug ts=2026-04-29T19:58:40.111604421Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=info ts=2026-04-29T19:58:43.110290307Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:58:43.110319403Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-29T19:58:43.110310066Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:58:43.110327053Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T19:58:43.487962854Z caller=checkpoint.go:372 msg="writing series" size="3.5 MB"
level=debug ts=2026-04-29T19:58:43.487984462Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.5 MB" series=1
ts=2026-04-29T19:58:43.841057604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:38786"
level=debug ts=2026-04-29T19:58:44.354304584Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-29T19:58:46.12673129Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T19:58:46.861271431Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=info ts=2026-04-29T19:58:46.910643374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="51 MB" avg_comp="2.8 MB" total_uncomp="416 MB" avg_uncomp="23 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:58:46.919197138Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T19:58:48.110824456Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:58:50.235956161Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=warn ts=2026-04-29T19:58:53.110289047Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:58:53.610704576Z caller=checkpoint.go:372 msg="writing series" size="378 kB"
level=debug ts=2026-04-29T19:58:56.986329316Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-04-29T19:58:57.662575725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:58:57.668349912Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-29T19:58:58.110866335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:58:59.986795897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T19:58:59.993953126Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-29T19:59:00.014070222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:42792"
level=debug ts=2026-04-29T19:59:00.36126846Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-04-29T19:59:03.110324898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:59:03.735462864Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
ts=2026-04-29T19:59:06.815001712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:56288"
level=debug ts=2026-04-29T19:59:07.111143448Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=warn ts=2026-04-29T19:59:08.110632999Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:59:08.628480093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T19:59:08.635954408Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T19:59:10.485980345Z caller=checkpoint.go:372 msg="writing series" size="268 kB"
level=info ts=2026-04-29T19:59:13.087916832Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T19:59:13.088362626Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T19:59:13.109777963Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:59:13.109804566Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T19:59:13.109812166Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T19:59:13.109890266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T19:59:13.844301814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:50198"
level=debug ts=2026-04-29T19:59:13.86101388Z caller=checkpoint.go:372 msg="writing series" size="201 kB"
level=debug ts=2026-04-29T19:59:13.861036671Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=debug ts=2026-04-29T19:59:14.793223284Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-29T19:59:16.129497806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T19:59:17.235889403Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-04-29T19:59:18.110839201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:59:19.482649576Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T19:59:20.610992837Z caller=checkpoint.go:372 msg="writing series" size="532 kB"
level=info ts=2026-04-29T19:59:21.466106021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="74 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:59:21.474595988Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:59:21.474619758Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T19:59:21.474645361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="74 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:59:21.480580632Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T19:59:23.110174026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:59:23.986402473Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=debug ts=2026-04-29T19:59:27.360759895Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=warn ts=2026-04-29T19:59:28.110664367Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:59:30.735715066Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-29T19:59:33.109846754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:59:34.111489204Z caller=checkpoint.go:372 msg="writing series" size="482 kB"
level=debug ts=2026-04-29T19:59:34.111512579Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=debug ts=2026-04-29T19:59:37.485543674Z caller=checkpoint.go:372 msg="writing series" size="404 kB"
level=warn ts=2026-04-29T19:59:38.110794757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:59:40.462212481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="74 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T19:59:40.473912718Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T19:59:40.861257809Z caller=checkpoint.go:372 msg="writing series" size="243 kB"
level=info ts=2026-04-29T19:59:41.504297005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="51 MB" avg_comp="2.8 MB" total_uncomp="416 MB" avg_uncomp="23 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T19:59:41.513229042Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T19:59:42.404854461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T19:59:42.41719446Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T19:59:42.417219949Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T19:59:42.417244103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T19:59:42.425974465Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-29T19:59:43.11025835Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T19:59:43.110289094Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-29T19:59:43.110277056Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:59:43.110298469Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T19:59:44.235981771Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
ts=2026-04-29T19:59:46.131810072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T19:59:46.993693126Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T19:59:47.611999233Z caller=checkpoint.go:372 msg="writing series" size="761 kB"
level=debug ts=2026-04-29T19:59:47.612030512Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=4
level=warn ts=2026-04-29T19:59:48.110778466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:59:50.986191236Z caller=checkpoint.go:372 msg="writing series" size="216 kB"
level=warn ts=2026-04-29T19:59:53.109847807Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T19:59:54.361021404Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-04-29T19:59:55.59690426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T19:59:55.609539007Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-29T19:59:57.387377684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T19:59:57.391499108Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T19:59:57.736192015Z caller=checkpoint.go:372 msg="writing series" size="745 kB"
level=warn ts=2026-04-29T19:59:58.110685686Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T19:59:58.390329368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T19:59:58.400185191Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-zciz\"}"
ts=2026-04-29T20:00:00.02066752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:39876"
level=debug ts=2026-04-29T20:00:01.111250359Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=debug ts=2026-04-29T20:00:01.111273072Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-04-29T20:00:03.110784275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:00:04.486018576Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
ts=2026-04-29T20:00:06.819497814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:34882"
level=debug ts=2026-04-29T20:00:07.863532563Z caller=checkpoint.go:372 msg="writing series" size="638 B"
level=warn ts=2026-04-29T20:00:08.110068942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:00:11.236635446Z caller=checkpoint.go:372 msg="writing series" size="622 kB"
level=debug ts=2026-04-29T20:00:13.087629419Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:00:13.088646209Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:00:13.089906018Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-29T20:00:13.089948363Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974991 n=1
level=debug ts=2026-04-29T20:00:13.089986247Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974991 n=1
level=info ts=2026-04-29T20:00:13.110142373Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:00:13.110171722Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-29T20:00:13.110158351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:00:13.110179303Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T20:00:13.329593508Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:00:13.849720578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:46300"
level=debug ts=2026-04-29T20:00:14.611076503Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-04-29T20:00:16.062539267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="74 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:00:16.079745046Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-29T20:00:16.134361631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:00:17.629772028Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:00:17.986426106Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-04-29T20:00:17.986461597Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-04-29T20:00:18.110745512Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:00:21.361056272Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=info ts=2026-04-29T20:00:22.63375101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="51 MB" avg_comp="2.8 MB" total_uncomp="416 MB" avg_uncomp="23 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:00:22.640900565Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T20:00:23.110093279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:00:24.735380476Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=warn ts=2026-04-29T20:00:28.110147564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:00:28.1180502Z caller=checkpoint.go:372 msg="writing series" size="7.6 MB"
level=debug ts=2026-04-29T20:00:28.118067678Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.9 MB" series=3
ts=2026-04-29T20:00:30.024583314Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:41114"
level=info ts=2026-04-29T20:00:31.41383592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:00:31.423926919Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:00:31.485853022Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-04-29T20:00:33.109954599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:00:34.864426144Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-04-29T20:00:34.867997955Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=2
ts=2026-04-29T20:00:36.821918187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:42484"
level=warn ts=2026-04-29T20:00:38.110357486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:00:38.235622514Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-04-29T20:00:40.466580155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:00:40.475878169Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:00:41.610712564Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=info ts=2026-04-29T20:00:43.110233988Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:00:43.110265811Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:00:43.110273365Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:00:43.110290175Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:00:44.986099248Z caller=checkpoint.go:372 msg="writing series" size="385 kB"
ts=2026-04-29T20:00:46.136998369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=warn ts=2026-04-29T20:00:48.110627293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:00:48.360617983Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=debug ts=2026-04-29T20:00:51.735902489Z caller=checkpoint.go:372 msg="writing series" size="268 kB"
level=debug ts=2026-04-29T20:00:52.331222099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T20:00:52.501152464Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T20:00:53.109839583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:00:53.947558027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:00:53.971652241Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T20:00:55.111500606Z caller=checkpoint.go:372 msg="writing series" size="332 kB"
level=debug ts=2026-04-29T20:00:55.111523151Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=warn ts=2026-04-29T20:00:58.110470282Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:00:58.495007014Z caller=checkpoint.go:372 msg="writing series" size="9.5 MB"
level=debug ts=2026-04-29T20:00:58.495031242Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.5 MB" series=1
ts=2026-04-29T20:01:00.027640115Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:35652"
level=debug ts=2026-04-29T20:01:01.861086815Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=warn ts=2026-04-29T20:01:03.110199307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:01:03.177291745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="74 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:01:03.186700114Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T20:01:05.235586891Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=warn ts=2026-04-29T20:01:08.110307216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:01:08.611300262Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=debug ts=2026-04-29T20:01:11.985835031Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-29T20:01:13.08793245Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:01:13.088539154Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:01:13.109979718Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:01:13.110011867Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:01:13.110029172Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:01:13.110044054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:01:13.472543454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="51 MB" avg_comp="2.8 MB" total_uncomp="416 MB" avg_uncomp="23 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:01:13.480948811Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:01:15.362637626Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
ts=2026-04-29T20:01:16.13990929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T20:01:17.916777856Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T20:01:18.111639314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:01:18.736205647Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-04-29T20:01:22.111996337Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-04-29T20:01:22.112018998Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=7
level=warn ts=2026-04-29T20:01:23.110563939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:01:25.485988803Z caller=checkpoint.go:372 msg="writing series" size="225 kB"
level=info ts=2026-04-29T20:01:26.732894155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:01:26.742384894Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-29T20:01:28.110333083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:01:28.861322315Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
ts=2026-04-29T20:01:30.030837446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:34852"
level=debug ts=2026-04-29T20:01:32.235731047Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-04-29T20:01:33.109915735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:01:34.927460966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:01:34.932240309Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:01:35.610591307Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
ts=2026-04-29T20:01:36.827105214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:57352"
level=warn ts=2026-04-29T20:01:38.110416723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:01:38.985680497Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=debug ts=2026-04-29T20:01:42.363551043Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-04-29T20:01:42.363578702Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=6
level=info ts=2026-04-29T20:01:43.11021737Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:01:43.11024637Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-29T20:01:43.110239465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:01:43.110254163Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-29T20:01:43.858592114Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:56706"
level=info ts=2026-04-29T20:01:44.917678149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="74 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:01:44.925534702Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T20:01:45.736038432Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-04-29T20:01:46.142788129Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T20:01:47.031092436Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T20:01:48.110555731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:01:49.110796695Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-04-29T20:01:52.486234131Z caller=checkpoint.go:372 msg="writing series" size="938 kB"
level=info ts=2026-04-29T20:01:52.687882694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:01:52.695332957Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-zciz\"}"
level=warn ts=2026-04-29T20:01:53.110335126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:01:54.077071509Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T20:01:55.860569838Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-04-29T20:01:58.110783137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:01:59.235712179Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=debug ts=2026-04-29T20:01:59.235735371Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=debug ts=2026-04-29T20:02:02.611193064Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-04-29T20:02:03.110835173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:02:05.986421237Z caller=checkpoint.go:372 msg="writing series" size="829 kB"
ts=2026-04-29T20:02:06.829697864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:40958"
level=info ts=2026-04-29T20:02:07.955618146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:02:07.973977394Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-29T20:02:08.110804043Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:02:09.360536333Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-04-29T20:02:12.721704315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="51 MB" avg_comp="2.8 MB" total_uncomp="416 MB" avg_uncomp="23 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:02:12.729374385Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:02:12.735415352Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=debug ts=2026-04-29T20:02:12.735430821Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-04-29T20:02:13.087923003Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:02:13.08835454Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:02:13.110353897Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:02:13.11038Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-29T20:02:13.110370435Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:02:13.110387356Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T20:02:16.11126444Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
ts=2026-04-29T20:02:16.146259965Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:02:16.295201122Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T20:02:18.110185101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:02:19.48636765Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-04-29T20:02:22.862403171Z caller=checkpoint.go:372 msg="writing series" size="347 kB"
level=warn ts=2026-04-29T20:02:23.110419971Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:02:26.235701639Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-04-29T20:02:28.110487974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:02:29.611575217Z caller=checkpoint.go:372 msg="writing series" size="627 kB"
level=debug ts=2026-04-29T20:02:29.611595373Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
ts=2026-04-29T20:02:30.036167014Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:57326"
level=debug ts=2026-04-29T20:02:32.986683217Z caller=checkpoint.go:372 msg="writing series" size="743 kB"
level=warn ts=2026-04-29T20:02:33.111551091Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:02:34.52599108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:02:34.529839413Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:02:34.529863057Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:02:34.529884387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:02:34.533709334Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T20:02:36.065198409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:02:36.073197142Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T20:02:36.201227545Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:02:36.203009839Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-29T20:02:36.360513972Z caller=checkpoint.go:372 msg="writing series" size="154 kB"
level=warn ts=2026-04-29T20:02:38.110019943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:02:38.726469468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="74 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:02:38.736796036Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T20:02:39.792282095Z caller=checkpoint.go:372 msg="writing series" size="53 MB"
level=debug ts=2026-04-29T20:02:39.792313429Z caller=checkpoint.go:384 msg="flushing series" totalSize="54 MB" series=3
level=info ts=2026-04-29T20:02:43.110388239Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T20:02:43.1104132Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:02:43.110427564Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:02:43.110447454Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:02:43.120133354Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-04-29T20:02:43.121793153Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.011489191s
level=debug ts=2026-04-29T20:02:45.242832783Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-29T20:02:46.149627171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=info ts=2026-04-29T20:02:47.710898845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:02:47.715253803Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-29T20:02:48.110677392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:02:51.149695309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:02:51.159795398Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-29T20:02:53.110323706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T20:02:58.110360809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:03:01.078158808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="51 MB" avg_comp="2.8 MB" total_uncomp="416 MB" avg_uncomp="23 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:03:01.084846278Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:03:01.084872862Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:03:01.084909221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="51 MB" avg_comp="2.8 MB" total_uncomp="416 MB" avg_uncomp="23 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:03:01.090692023Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T20:03:03.110649973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T20:03:08.110917161Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:03:12.405920753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="51 MB" avg_comp="2.8 MB" total_uncomp="416 MB" avg_uncomp="23 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:03:12.415581163Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T20:03:13.08847742Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:03:13.088906742Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:03:13.110378451Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-29T20:03:13.110396771Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T20:03:13.110396286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:03:13.110421419Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:03:13.110445109Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:03:13.110565395Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=info ts=2026-04-29T20:03:13.110574473Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-29T20:03:13.110591675Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.608µs success=true
level=debug ts=2026-04-29T20:03:13.110798121Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-04-29T20:03:14.452160344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:03:14.456004602Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:03:14.493481545Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:03:14.493512539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="671 kB" avg_comp="671 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:03:14.497020385Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-04-29T20:03:16.152347937Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:03:16.486984962Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=warn ts=2026-04-29T20:03:18.110069713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:03:19.861386051Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-04-29T20:03:23.110202364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:03:23.236733257Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-04-29T20:03:23.860477028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:03:23.867104179Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T20:03:25.712232625Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T20:03:26.611129433Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=warn ts=2026-04-29T20:03:28.110246976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:03:28.659750161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:03:28.664671035Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:03:29.986170194Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
ts=2026-04-29T20:03:30.042318413Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:47410"
level=info ts=2026-04-29T20:03:30.934630219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="74 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:03:30.941231327Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T20:03:33.110552964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:03:33.361003142Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=debug ts=2026-04-29T20:03:36.736077117Z caller=checkpoint.go:372 msg="writing series" size="3.6 kB"
level=warn ts=2026-04-29T20:03:38.110670988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:03:40.111756704Z caller=checkpoint.go:372 msg="writing series" size="561 kB"
level=debug ts=2026-04-29T20:03:40.111781008Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=info ts=2026-04-29T20:03:40.169567017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="54 MB" avg_comp="2.8 MB" total_uncomp="440 MB" avg_uncomp="23 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:03:40.178036744Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T20:03:43.110457602Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T20:03:43.110468421Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:03:43.110493797Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:03:43.110506776Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T20:03:43.489020476Z caller=checkpoint.go:372 msg="writing series" size="3.6 MB"
level=debug ts=2026-04-29T20:03:43.489052124Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.6 MB" series=1
level=info ts=2026-04-29T20:03:43.55953818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:03:43.56985069Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-29T20:03:46.155805435Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T20:03:46.861363245Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=debug ts=2026-04-29T20:03:47.2286993Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:03:47.228730364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:03:47.232785769Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-04-29T20:03:48.11084306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:03:50.236288908Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=warn ts=2026-04-29T20:03:53.110320938Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:03:53.611160846Z caller=checkpoint.go:372 msg="writing series" size="622 kB"
level=debug ts=2026-04-29T20:03:56.985905323Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-29T20:03:58.11042561Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:04:00.361273084Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-04-29T20:04:02.293188191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:04:02.298676721Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-04-29T20:04:03.110468194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:04:03.736053787Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-04-29T20:04:06.64209548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:04:06.646613305Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:04:07.111566642Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=warn ts=2026-04-29T20:04:08.110106042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:04:10.48675053Z caller=checkpoint.go:372 msg="writing series" size="268 kB"
level=debug ts=2026-04-29T20:04:10.48677089Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-04-29T20:04:12.635392468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:04:12.640211365Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T20:04:13.087893452Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:04:13.08789968Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:04:13.110473733Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:04:13.110498153Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:04:13.11050799Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:04:13.110523196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:04:13.861790409Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
ts=2026-04-29T20:04:13.868991148Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:44120"
ts=2026-04-29T20:04:16.159239747Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:04:17.236851951Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-04-29T20:04:18.110071129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:04:18.579338522Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:04:19.644635259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:04:19.653417348Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:04:19.653455623Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:04:19.653481178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:04:19.66132965Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T20:04:20.6113445Z caller=checkpoint.go:372 msg="writing series" size="532 kB"
level=info ts=2026-04-29T20:04:22.439896522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="74 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:04:22.447920619Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T20:04:23.110167981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:04:23.986818923Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=debug ts=2026-04-29T20:04:26.263502152Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T20:04:27.361802378Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=warn ts=2026-04-29T20:04:28.110229936Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T20:04:30.047533475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:40256"
level=debug ts=2026-04-29T20:04:30.736053852Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-29T20:04:33.109971483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:04:33.766477811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:04:33.77141355Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:04:34.112032812Z caller=checkpoint.go:372 msg="writing series" size="397 kB"
level=debug ts=2026-04-29T20:04:34.112055069Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=info ts=2026-04-29T20:04:34.141933808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="54 MB" avg_comp="2.8 MB" total_uncomp="440 MB" avg_uncomp="23 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:04:34.149357344Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:04:37.48699552Z caller=checkpoint.go:372 msg="writing series" size="313 kB"
level=warn ts=2026-04-29T20:04:38.110094992Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:04:38.548942874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:04:38.558489821Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T20:04:40.861572977Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
level=info ts=2026-04-29T20:04:40.960888085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:04:40.967543952Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-29T20:04:43.110277211Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:04:43.110307137Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:04:43.110314199Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:04:43.110309194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T20:04:43.872324471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:44022"
level=debug ts=2026-04-29T20:04:43.922340779Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:04:44.236356793Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
ts=2026-04-29T20:04:46.162254289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:04:47.612044227Z caller=checkpoint.go:372 msg="writing series" size="761 kB"
level=debug ts=2026-04-29T20:04:47.61206841Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=warn ts=2026-04-29T20:04:48.110056059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:04:50.986842894Z caller=checkpoint.go:372 msg="writing series" size="216 kB"
level=warn ts=2026-04-29T20:04:53.110494666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:04:54.361575203Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-04-29T20:04:57.325223455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:04:57.332588487Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-29T20:04:57.592338566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:04:57.596488176Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:04:57.736121151Z caller=checkpoint.go:372 msg="writing series" size="745 kB"
level=warn ts=2026-04-29T20:04:58.110010344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:05:01.110893256Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=debug ts=2026-04-29T20:05:01.110916744Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-04-29T20:05:03.110708533Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:05:04.486664673Z caller=checkpoint.go:372 msg="writing series" size="234 kB"
ts=2026-04-29T20:05:06.842084953Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:42880"
level=debug ts=2026-04-29T20:05:07.861508329Z caller=checkpoint.go:372 msg="writing series" size="627 B"
level=warn ts=2026-04-29T20:05:08.110063916Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:05:10.902809125Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T20:05:11.23686139Z caller=checkpoint.go:372 msg="writing series" size="622 kB"
level=info ts=2026-04-29T20:05:13.088036659Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:05:13.088509426Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:05:13.109936501Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T20:05:13.109956754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:05:13.109969251Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:05:13.109979892Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-29T20:05:13.875129157Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:49906"
level=info ts=2026-04-29T20:05:14.06024543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:05:14.065229008Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:05:14.61114614Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-04-29T20:05:15.110948544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:05:15.117233974Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-zciz\"}"
ts=2026-04-29T20:05:16.165182732Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T20:05:17.724179095Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:05:17.985949404Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=debug ts=2026-04-29T20:05:17.985973023Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-04-29T20:05:18.110472625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:05:21.361929472Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
level=info ts=2026-04-29T20:05:22.032725203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="74 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:05:22.039685429Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T20:05:23.110334658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:05:23.845499554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="54 MB" avg_comp="2.8 MB" total_uncomp="440 MB" avg_uncomp="23 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:05:23.852106828Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:05:24.736140084Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=warn ts=2026-04-29T20:05:28.110735419Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:05:28.11985592Z caller=checkpoint.go:372 msg="writing series" size="9.4 MB"
level=debug ts=2026-04-29T20:05:28.119875865Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.7 MB" series=3
level=info ts=2026-04-29T20:05:31.026475637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:05:31.034325002Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:05:31.486014609Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-04-29T20:05:33.110238792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:05:34.863105681Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-04-29T20:05:34.863128604Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=2
level=warn ts=2026-04-29T20:05:38.109918835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:05:38.236207644Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-04-29T20:05:41.611725576Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=info ts=2026-04-29T20:05:43.11037736Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:05:43.110405222Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:05:43.110412667Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:05:43.110430473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:05:44.022269087Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:05:44.986796457Z caller=checkpoint.go:372 msg="writing series" size="464 kB"
ts=2026-04-29T20:05:46.168767318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=info ts=2026-04-29T20:05:46.924490062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:05:46.930523466Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-29T20:05:48.110529534Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:05:48.361628969Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=debug ts=2026-04-29T20:05:51.736465535Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=warn ts=2026-04-29T20:05:53.110487394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:05:55.111721607Z caller=checkpoint.go:372 msg="writing series" size="160 kB"
level=info ts=2026-04-29T20:05:57.223771695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:05:57.230665046Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-29T20:05:58.110157974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:05:58.495126823Z caller=checkpoint.go:372 msg="writing series" size="10 MB"
level=debug ts=2026-04-29T20:05:58.49514958Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=7
level=debug ts=2026-04-29T20:06:01.861196231Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=info ts=2026-04-29T20:06:02.213464543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:06:02.223101288Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-zciz\"}"
level=warn ts=2026-04-29T20:06:03.110520634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:06:04.280257006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:06:04.28440909Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:06:05.23593199Z caller=checkpoint.go:372 msg="writing series" size="935 B"
ts=2026-04-29T20:06:06.846397815Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:58332"
level=warn ts=2026-04-29T20:06:08.110487775Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:06:08.61188069Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=debug ts=2026-04-29T20:06:11.324236789Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T20:06:11.986471034Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-04-29T20:06:13.087696304Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:06:13.088772889Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-29T20:06:13.11035951Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:06:13.110388065Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-29T20:06:13.110374461Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:06:13.110396259Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:06:14.138278186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="54 MB" avg_comp="2.8 MB" total_uncomp="440 MB" avg_uncomp="23 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:06:14.145797327Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:06:15.361491293Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
ts=2026-04-29T20:06:16.172385092Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T20:06:17.533946863Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T20:06:18.110726532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:06:18.415558601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="74 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:06:18.422564023Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:06:18.422587207Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:06:18.422610478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="74 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:06:18.428273691Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T20:06:18.736767347Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-04-29T20:06:22.1125611Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-04-29T20:06:22.112585077Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=7
level=warn ts=2026-04-29T20:06:23.110590339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:06:24.416652701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:06:24.424805521Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:06:24.424829643Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:06:24.424853594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:06:24.430628677Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:06:25.487015938Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=warn ts=2026-04-29T20:06:28.110629913Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:06:28.860983601Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
level=info ts=2026-04-29T20:06:29.686891273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:06:29.691635229Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:06:32.236161621Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-04-29T20:06:33.110028582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:06:35.61149726Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
level=info ts=2026-04-29T20:06:36.105117763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="74 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:06:36.114223352Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:06:38.055565016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:06:38.065217028Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-29T20:06:38.109899685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:06:38.985844009Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=info ts=2026-04-29T20:06:42.299190811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:06:42.304884362Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:06:42.362842885Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-04-29T20:06:42.362867856Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=6
level=debug ts=2026-04-29T20:06:42.940555582Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:06:43.110758814Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:06:43.110792991Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:06:43.110801556Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:06:43.110803414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:06:45.736832964Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
ts=2026-04-29T20:06:46.175530222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=info ts=2026-04-29T20:06:47.055249684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:06:47.062027846Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-zciz\"}"
level=warn ts=2026-04-29T20:06:48.110599715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:06:49.111483142Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=debug ts=2026-04-29T20:06:52.486911655Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-29T20:06:52.48693397Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=warn ts=2026-04-29T20:06:53.11041345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:06:55.861519471Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-04-29T20:06:58.110471327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:06:59.236459243Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=info ts=2026-04-29T20:06:59.991495394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="54 MB" avg_comp="2.8 MB" total_uncomp="440 MB" avg_uncomp="23 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:07:00.003209231Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-29T20:07:00.059524236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:33526"
level=info ts=2026-04-29T20:07:01.283606994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="74 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:07:01.29070037Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T20:07:02.611488941Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-04-29T20:07:03.110349773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:07:03.280838961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:07:03.284787655Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:07:05.986969981Z caller=checkpoint.go:372 msg="writing series" size="846 kB"
level=debug ts=2026-04-29T20:07:05.986996807Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=debug ts=2026-04-29T20:07:07.514216469Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:07:07.516026828Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T20:07:08.110712623Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:07:09.361493237Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-29T20:07:12.000244932Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:07:12.736525274Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=debug ts=2026-04-29T20:07:13.088275992Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:07:13.088285418Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-29T20:07:13.109785007Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:07:13.109808003Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:07:13.109815232Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:07:13.109900292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:07:16.111811427Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
ts=2026-04-29T20:07:16.17877795Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=info ts=2026-04-29T20:07:16.979154665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:07:16.987523435Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-29T20:07:18.111765317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:07:19.486209112Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-04-29T20:07:22.861408902Z caller=checkpoint.go:372 msg="writing series" size="347 kB"
level=warn ts=2026-04-29T20:07:23.11031888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:07:25.161819419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:07:25.16628046Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:07:26.236381981Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-04-29T20:07:28.110026361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:07:29.611986632Z caller=checkpoint.go:372 msg="writing series" size="627 kB"
level=debug ts=2026-04-29T20:07:29.612006021Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=7
level=debug ts=2026-04-29T20:07:32.986648237Z caller=checkpoint.go:372 msg="writing series" size="825 kB"
level=warn ts=2026-04-29T20:07:33.110634806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:07:36.361291761Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
level=warn ts=2026-04-29T20:07:38.110144452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:07:39.788626777Z caller=checkpoint.go:372 msg="writing series" size="56 MB"
level=debug ts=2026-04-29T20:07:39.788652524Z caller=checkpoint.go:384 msg="flushing series" totalSize="57 MB" series=3
level=info ts=2026-04-29T20:07:40.33684336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:07:40.344755167Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:07:42.076588538Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:07:43.110418634Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:07:43.110455909Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:07:43.110464342Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:07:43.110491976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:07:43.125196189Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-04-29T20:07:43.12670287Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.015977636s
level=info ts=2026-04-29T20:07:43.217063486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:07:43.225487845Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T20:07:43.369854081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:07:43.373666155Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-29T20:07:45.985678947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="54 MB" avg_comp="2.8 MB" total_uncomp="440 MB" avg_uncomp="23 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:07:45.992924956Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-29T20:07:46.181872661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=warn ts=2026-04-29T20:07:48.11056548Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T20:07:53.11048249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:07:55.320361303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="74 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:07:55.328840471Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T20:07:57.756089593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-29T20:07:58.081057199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:07:58.088747091Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-29T20:07:58.109899508Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-29T20:08:03.110495995Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:08:08.010279701Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T20:08:08.110220941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:08:13.08891399Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:08:13.089899177Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:08:13.11028239Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-04-29T20:08:13.110295594Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:08:13.110301803Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:08:13.110328994Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:08:13.11033873Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:08:13.110473015Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=info ts=2026-04-29T20:08:13.110484526Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-29T20:08:13.110508307Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.18µs success=true
level=debug ts=2026-04-29T20:08:13.110707883Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
ts=2026-04-29T20:08:16.185302188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:08:16.48669604Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=warn ts=2026-04-29T20:08:18.110092243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:08:19.005824389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:08:19.014123545Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:08:19.861081209Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-04-29T20:08:23.109885551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:08:23.236266919Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-04-29T20:08:25.611911962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:08:25.61634872Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:08:26.6112792Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-04-29T20:08:27.476406695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:08:27.483667592Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=warn ts=2026-04-29T20:08:28.110611426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:08:29.986280388Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-04-29T20:08:31.927196021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:08:31.930714345Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-04-29T20:08:33.109833363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:08:33.361348361Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=info ts=2026-04-29T20:08:34.762724908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="54 MB" avg_comp="2.8 MB" total_uncomp="440 MB" avg_uncomp="23 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:08:34.769241101Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:08:36.736072465Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=info ts=2026-04-29T20:08:36.975847363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="74 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:08:36.983516309Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T20:08:38.110487621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:08:39.410089561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:08:39.417658277Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:08:40.111734049Z caller=checkpoint.go:372 msg="writing series" size="511 kB"
level=debug ts=2026-04-29T20:08:40.111755379Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=debug ts=2026-04-29T20:08:42.74680489Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:08:43.110300519Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:08:43.110329833Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:08:43.110337638Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:08:43.110330408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:08:43.489260914Z caller=checkpoint.go:372 msg="writing series" size="3.7 MB"
level=debug ts=2026-04-29T20:08:43.489281252Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.7 MB" series=1
ts=2026-04-29T20:08:43.888186228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:44848"
ts=2026-04-29T20:08:46.187582501Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T20:08:46.861703269Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-04-29T20:08:48.11041551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:08:50.236593015Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=warn ts=2026-04-29T20:08:53.110300569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:08:53.612083776Z caller=checkpoint.go:372 msg="writing series" size="512 kB"
level=debug ts=2026-04-29T20:08:56.98665316Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-29T20:08:58.110518212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:08:59.345083434Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:09:00.068653558Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:44446"
level=debug ts=2026-04-29T20:09:00.361253299Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-04-29T20:09:00.424483007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:09:00.428471357Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-29T20:09:03.10993434Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:09:03.73623292Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-04-29T20:09:06.30362868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:09:06.307830649Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:09:06.30785306Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:09:06.307877835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:09:06.311242956Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:09:07.111363819Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=warn ts=2026-04-29T20:09:08.110313412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:09:08.260701687Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:09:10.486324277Z caller=checkpoint.go:372 msg="writing series" size="268 kB"
level=debug ts=2026-04-29T20:09:10.486352587Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-04-29T20:09:11.509202829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:09:11.517554412Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T20:09:13.088273356Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:09:13.088797027Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:09:13.110212685Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:09:13.110235276Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-29T20:09:13.110226945Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:09:13.11024239Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T20:09:13.861142599Z caller=checkpoint.go:372 msg="writing series" size="308 kB"
ts=2026-04-29T20:09:16.190062999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T20:09:17.236380025Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-04-29T20:09:17.678647444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:09:17.682953688Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-29T20:09:18.110087121Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:09:18.127242174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="74 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:09:18.134952153Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:09:20.045954721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:09:20.053374822Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:09:20.611583553Z caller=checkpoint.go:372 msg="writing series" size="532 kB"
level=warn ts=2026-04-29T20:09:23.110473532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:09:23.98667544Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=info ts=2026-04-29T20:09:24.407260736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:09:24.41104467Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:09:27.360989149Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=warn ts=2026-04-29T20:09:28.110021337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:09:29.459583778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="54 MB" avg_comp="2.8 MB" total_uncomp="440 MB" avg_uncomp="23 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:09:29.469851922Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:09:29.46987277Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:09:29.469909045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="54 MB" avg_comp="2.8 MB" total_uncomp="440 MB" avg_uncomp="23 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:09:29.479074318Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-29T20:09:30.072206134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:52316"
level=debug ts=2026-04-29T20:09:30.736015933Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-29T20:09:33.110159555Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:09:34.111913906Z caller=checkpoint.go:372 msg="writing series" size="498 kB"
level=debug ts=2026-04-29T20:09:34.111934939Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=7
level=debug ts=2026-04-29T20:09:37.486626776Z caller=checkpoint.go:372 msg="writing series" size="425 kB"
level=warn ts=2026-04-29T20:09:38.110244666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:09:40.861859245Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=debug ts=2026-04-29T20:09:41.645321398Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:09:43.110348674Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:09:43.1103761Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:09:43.110383733Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:09:43.110396541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:09:43.492891184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:09:43.500186087Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:09:43.500207719Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:09:43.500228615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:09:43.503628641Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:09:44.236650845Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
ts=2026-04-29T20:09:46.193380697Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T20:09:46.506646488Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-29T20:09:47.184787674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:09:47.191349146Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:09:47.612179917Z caller=checkpoint.go:372 msg="writing series" size="761 kB"
level=debug ts=2026-04-29T20:09:47.612202187Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=4
level=warn ts=2026-04-29T20:09:48.110062931Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:09:50.98686439Z caller=checkpoint.go:372 msg="writing series" size="262 kB"
level=info ts=2026-04-29T20:09:51.863067852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:09:51.874904252Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-29T20:09:53.110270531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:09:54.361480971Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-04-29T20:09:57.7368629Z caller=checkpoint.go:372 msg="writing series" size="745 kB"
level=warn ts=2026-04-29T20:09:58.110547034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:09:58.285602037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:09:58.293500054Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
ts=2026-04-29T20:10:00.07546538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:54708"
level=debug ts=2026-04-29T20:10:01.111378862Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=debug ts=2026-04-29T20:10:01.111402022Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-04-29T20:10:01.624744668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:10:01.629776456Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-29T20:10:03.110301509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:10:04.486099933Z caller=checkpoint.go:372 msg="writing series" size="266 kB"
level=info ts=2026-04-29T20:10:06.844666424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="74 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:10:06.853670877Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-29T20:10:06.861520616Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:38570"
level=debug ts=2026-04-29T20:10:07.860988485Z caller=checkpoint.go:372 msg="writing series" size="653 B"
level=warn ts=2026-04-29T20:10:08.109856933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:10:08.690257023Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:10:11.236092252Z caller=checkpoint.go:372 msg="writing series" size="735 kB"
level=info ts=2026-04-29T20:10:13.087919295Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:10:13.087934772Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:10:13.110365397Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T20:10:13.110369578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:10:13.110390328Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:10:13.110398719Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T20:10:14.611205781Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-04-29T20:10:14.611223652Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
ts=2026-04-29T20:10:16.196803927Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=info ts=2026-04-29T20:10:17.06424982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:10:17.072628055Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:10:17.986562721Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=warn ts=2026-04-29T20:10:18.109905063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:10:19.469843765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:10:19.473564648Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:10:19.473587184Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:10:19.473607613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:10:19.476367782Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:10:21.361215147Z caller=checkpoint.go:372 msg="writing series" size="309 kB"
level=info ts=2026-04-29T20:10:22.320023428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:10:22.326699235Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T20:10:23.110200388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:10:24.736428409Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=warn ts=2026-04-29T20:10:28.110227425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:10:28.12285367Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-04-29T20:10:28.122874775Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=4
level=debug ts=2026-04-29T20:10:29.001190423Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T20:10:31.486737582Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-04-29T20:10:32.448363555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:10:32.454149055Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-29T20:10:33.11070269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:10:33.438707206Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:10:34.86328253Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-04-29T20:10:34.863303844Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=2
level=warn ts=2026-04-29T20:10:38.110680108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:10:38.235995664Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-04-29T20:10:39.352927638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:10:39.357035666Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-29T20:10:41.606144883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:10:41.610265683Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:10:41.611301855Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=info ts=2026-04-29T20:10:43.109868965Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:10:43.109902155Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:10:43.109911332Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:10:43.109898896Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T20:10:43.897106588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:51762"
level=debug ts=2026-04-29T20:10:44.986187972Z caller=checkpoint.go:372 msg="writing series" size="409 kB"
ts=2026-04-29T20:10:46.200078889Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=warn ts=2026-04-29T20:10:48.112514939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:10:48.361488744Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=debug ts=2026-04-29T20:10:51.736227451Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=warn ts=2026-04-29T20:10:53.109998243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:10:55.018651517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="74 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:10:55.027846108Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T20:10:55.111529023Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=info ts=2026-04-29T20:10:58.012723647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:10:58.019537749Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:10:58.019558317Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:10:58.019582218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:10:58.025661803Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=warn ts=2026-04-29T20:10:58.110645853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:10:58.495172443Z caller=checkpoint.go:372 msg="writing series" size="10 MB"
level=debug ts=2026-04-29T20:10:58.495195116Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=7
ts=2026-04-29T20:11:00.081639703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:44362"
level=debug ts=2026-04-29T20:11:01.861565214Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=warn ts=2026-04-29T20:11:03.110671405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:11:04.149456732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:11:04.157339387Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:11:05.236348172Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=debug ts=2026-04-29T20:11:06.733529919Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:11:08.000273668Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:11:08.002001228Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T20:11:08.110269477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:11:08.611074189Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-04-29T20:11:10.403342848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:11:10.408318871Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:11:11.986094519Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-29T20:11:13.087943819Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:11:13.088171996Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:11:13.110559925Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:11:13.110584098Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:11:13.110592515Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:11:13.110607933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:11:13.895928655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:11:13.903477445Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T20:11:14.700950901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:11:14.707814512Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:11:15.361026465Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
ts=2026-04-29T20:11:16.202688909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=warn ts=2026-04-29T20:11:18.110465052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:11:18.736003423Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-04-29T20:11:22.112567522Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-04-29T20:11:22.112589602Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=7
level=warn ts=2026-04-29T20:11:23.110445935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:11:25.486263767Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=info ts=2026-04-29T20:11:25.953470279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:11:25.959419957Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-29T20:11:28.110148897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:11:28.860869392Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
level=info ts=2026-04-29T20:11:31.31168841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:11:31.315788116Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:11:32.236376136Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=debug ts=2026-04-29T20:11:33.03992271Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T20:11:33.110285945Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:11:35.611675949Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
level=warn ts=2026-04-29T20:11:38.110423139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:11:38.986659787Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=info ts=2026-04-29T20:11:40.681321366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:11:40.689995572Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T20:11:42.363460651Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-04-29T20:11:42.363483438Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=6
level=info ts=2026-04-29T20:11:43.110169484Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:11:43.110201786Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-29T20:11:43.110188801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:11:43.110209767Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T20:11:45.736512001Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
ts=2026-04-29T20:11:46.205313042Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:11:47.783723246Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-29T20:11:47.807781006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:11:47.814928618Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=warn ts=2026-04-29T20:11:48.110451232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:11:49.110756445Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=debug ts=2026-04-29T20:11:52.486682738Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-04-29T20:11:52.486703898Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=warn ts=2026-04-29T20:11:53.10994359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:11:55.861395741Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-04-29T20:11:57.289678869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:11:57.293867893Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-04-29T20:11:58.110810502Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:11:59.236077061Z caller=checkpoint.go:372 msg="writing series" size="221 kB"
ts=2026-04-29T20:12:00.087150032Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:35576"
level=info ts=2026-04-29T20:12:01.834502579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:12:01.841464127Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T20:12:01.941845088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:12:01.950017643Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:12:02.611561283Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-04-29T20:12:03.110409627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:12:05.987136549Z caller=checkpoint.go:372 msg="writing series" size="930 kB"
level=debug ts=2026-04-29T20:12:05.98715971Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=warn ts=2026-04-29T20:12:08.110920879Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:12:08.183263785Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:12:09.361731237Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-04-29T20:12:12.448549795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:12:12.456325678Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:12:12.73630225Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-04-29T20:12:13.087935807Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:12:13.088447829Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:12:13.110772772Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:12:13.110795011Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:12:13.110801709Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:12:13.11082098Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:12:16.111611992Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
ts=2026-04-29T20:12:16.208562965Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=warn ts=2026-04-29T20:12:18.110783896Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:12:18.315087122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:12:18.319099223Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:12:19.486477993Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-04-29T20:12:22.861429604Z caller=checkpoint.go:372 msg="writing series" size="347 kB"
level=warn ts=2026-04-29T20:12:23.11041707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:12:26.236359607Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-04-29T20:12:28.110163847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:12:29.61189392Z caller=checkpoint.go:372 msg="writing series" size="627 kB"
level=debug ts=2026-04-29T20:12:29.611914929Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=7
level=debug ts=2026-04-29T20:12:32.986906651Z caller=checkpoint.go:372 msg="writing series" size="927 kB"
level=warn ts=2026-04-29T20:12:33.110081328Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:12:34.333165025Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:12:35.66567399Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-29T20:12:35.874270649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:12:35.883382694Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T20:12:36.35064176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:12:36.357048611Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:12:36.35706817Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:12:36.357091763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T20:12:36.361317866Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=debug ts=2026-04-29T20:12:36.3613341Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=error ts=2026-04-29T20:12:36.362846108Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-29T20:12:36.8727004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:55770"
level=warn ts=2026-04-29T20:12:38.110885626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:12:39.788554348Z caller=checkpoint.go:372 msg="writing series" size="57 MB"
level=debug ts=2026-04-29T20:12:39.788582597Z caller=checkpoint.go:384 msg="flushing series" totalSize="57 MB" series=1
level=info ts=2026-04-29T20:12:42.184998882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:12:42.19272399Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:12:42.192750821Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:12:42.192775215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:12:42.199644766Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-29T20:12:43.11021979Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:12:43.110249222Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:12:43.110257892Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:12:43.110295673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:12:43.121692201Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-04-29T20:12:43.123220671Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.012573126s
ts=2026-04-29T20:12:43.905139314Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:50112"
ts=2026-04-29T20:12:46.211174752Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=warn ts=2026-04-29T20:12:48.10994269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:12:48.338008398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:12:48.349068606Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:12:52.748260065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:12:52.75196417Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-04-29T20:12:53.110625283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:12:57.75558181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:12:57.763920274Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T20:12:58.110122286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:12:59.641622517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:12:59.649341821Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-29T20:13:03.110299632Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:13:06.014199102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:13:06.018208504Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-04-29T20:13:06.875535828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:58228"
level=debug ts=2026-04-29T20:13:07.327487689Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T20:13:08.110543694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:13:11.278770799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:13:11.282903908Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:13:13.087891583Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:13:13.088924693Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-29T20:13:13.110287349Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:13:13.110308935Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:13:13.110317455Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:13:13.110317068Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-04-29T20:13:13.110320491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:13:13.110476361Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=info ts=2026-04-29T20:13:13.110485213Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-29T20:13:13.110502963Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.693µs success=true
level=debug ts=2026-04-29T20:13:13.110707001Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
ts=2026-04-29T20:13:16.21342524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T20:13:16.486949485Z caller=checkpoint.go:372 msg="writing series" size="273 kB"
level=warn ts=2026-04-29T20:13:18.110617705Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:13:19.262783843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:13:19.274412675Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T20:13:19.861065566Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=debug ts=2026-04-29T20:13:22.050910676Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-29T20:13:23.109886303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:13:23.236232458Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-04-29T20:13:26.611211798Z caller=checkpoint.go:372 msg="writing series" size="71 kB"
level=info ts=2026-04-29T20:13:27.663177731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:13:27.670671066Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T20:13:28.110300157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:13:29.986039467Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
ts=2026-04-29T20:13:30.094271269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:52004"
level=warn ts=2026-04-29T20:13:33.110109536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:13:33.361533569Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=info ts=2026-04-29T20:13:36.624901629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:13:36.633893838Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:13:36.736297009Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
ts=2026-04-29T20:13:36.878559793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:33706"
level=warn ts=2026-04-29T20:13:38.110781091Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:13:38.634471876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:13:38.638283071Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:13:40.111910917Z caller=checkpoint.go:372 msg="writing series" size="587 kB"
level=debug ts=2026-04-29T20:13:40.111931181Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=9
level=debug ts=2026-04-29T20:13:40.8104299Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:13:43.109791581Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:13:43.109824248Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:13:43.109838601Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:13:43.109866732Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:13:43.490299766Z caller=checkpoint.go:372 msg="writing series" size="4.0 MB"
level=debug ts=2026-04-29T20:13:43.490319866Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.0 MB" series=1
ts=2026-04-29T20:13:43.9099771Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:33120"
ts=2026-04-29T20:13:46.216231098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:13:46.861261315Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-04-29T20:13:48.110244632Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:13:50.23602809Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=warn ts=2026-04-29T20:13:53.110534296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:13:53.612004449Z caller=checkpoint.go:372 msg="writing series" size="542 kB"
level=debug ts=2026-04-29T20:13:56.986500431Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-04-29T20:13:57.15510188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:13:57.162190383Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-29T20:13:57.534827892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:13:57.542166685Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T20:13:57.906274043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:13:57.910369853Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-29T20:13:58.109978978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T20:14:00.097094844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:36450"
level=debug ts=2026-04-29T20:14:00.361333981Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-04-29T20:14:03.110655262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:14:03.73596812Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-04-29T20:14:05.331487848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:14:05.339393912Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T20:14:07.111399555Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
level=warn ts=2026-04-29T20:14:08.110109379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:14:09.151798804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:14:09.160272409Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T20:14:10.48616439Z caller=checkpoint.go:372 msg="writing series" size="268 kB"
level=debug ts=2026-04-29T20:14:10.486188399Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=debug ts=2026-04-29T20:14:10.60140758Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:14:13.08796601Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:14:13.08798399Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:14:13.110451772Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:14:13.110484612Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:14:13.110495486Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:14:13.110516087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:14:13.861345865Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
ts=2026-04-29T20:14:16.219054912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:14:17.235878902Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-04-29T20:14:17.838654837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:14:17.846341109Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-29T20:14:18.110858509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:14:20.612280855Z caller=checkpoint.go:372 msg="writing series" size="532 kB"
level=debug ts=2026-04-29T20:14:20.88109397Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-29T20:14:23.110670608Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:14:23.986737668Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
level=debug ts=2026-04-29T20:14:27.360922549Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=warn ts=2026-04-29T20:14:28.110076414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T20:14:30.099835124Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:53686"
level=debug ts=2026-04-29T20:14:30.736569397Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-29T20:14:33.110017208Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:14:33.649140801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:14:33.655236599Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:14:34.111230431Z caller=checkpoint.go:372 msg="writing series" size="428 kB"
level=debug ts=2026-04-29T20:14:34.111250929Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
ts=2026-04-29T20:14:36.883157783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:47932"
level=debug ts=2026-04-29T20:14:37.486661392Z caller=checkpoint.go:372 msg="writing series" size="334 kB"
level=warn ts=2026-04-29T20:14:38.110036019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:14:40.861249498Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=debug ts=2026-04-29T20:14:40.961643411Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:14:43.110123054Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:14:43.110155229Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:14:43.11016491Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:14:43.110207709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:14:44.236145664Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
ts=2026-04-29T20:14:46.221990636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:14:47.61167718Z caller=checkpoint.go:372 msg="writing series" size="761 kB"
level=debug ts=2026-04-29T20:14:47.611698207Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=4
level=info ts=2026-04-29T20:14:47.825720225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:14:47.83279656Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-29T20:14:48.109856324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:14:48.891384106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:14:48.895497042Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:14:50.986420815Z caller=checkpoint.go:372 msg="writing series" size="262 kB"
level=warn ts=2026-04-29T20:14:53.110370993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:14:54.361342637Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-04-29T20:14:56.237893097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:14:56.245346311Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:14:57.736948565Z caller=checkpoint.go:372 msg="writing series" size="745 kB"
level=warn ts=2026-04-29T20:14:58.110285484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:14:58.961960119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:14:58.969187393Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T20:15:00.597646125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:15:00.604759959Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T20:15:01.111555887Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=debug ts=2026-04-29T20:15:01.111589067Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-04-29T20:15:03.110852548Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:15:04.485987084Z caller=checkpoint.go:372 msg="writing series" size="298 kB"
level=debug ts=2026-04-29T20:15:05.478922562Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:15:05.480635129Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-29T20:15:07.861580522Z caller=checkpoint.go:372 msg="writing series" size="653 B"
level=warn ts=2026-04-29T20:15:08.110389306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:15:11.237105337Z caller=checkpoint.go:372 msg="writing series" size="735 kB"
level=info ts=2026-04-29T20:15:13.087906158Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:15:13.088274938Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=debug ts=2026-04-29T20:15:13.089599338Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-29T20:15:13.089645225Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974992 n=1
level=debug ts=2026-04-29T20:15:13.089685896Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974992 n=1
level=info ts=2026-04-29T20:15:13.110612908Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:15:13.11063769Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:15:13.110645188Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:15:13.11069023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:15:13.439530056Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:15:14.611514443Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-04-29T20:15:14.611534188Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-04-29T20:15:16.095547504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:15:16.102407592Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-29T20:15:16.224942192Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T20:15:17.986588254Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=warn ts=2026-04-29T20:15:18.110173876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:15:21.361804518Z caller=checkpoint.go:372 msg="writing series" size="309 kB"
level=warn ts=2026-04-29T20:15:23.110228733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:15:24.389621357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:15:24.39425029Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:15:24.736127242Z caller=checkpoint.go:372 msg="writing series" size="6.7 kB"
level=warn ts=2026-04-29T20:15:28.110136768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:15:28.124347837Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-04-29T20:15:28.124363715Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=4
ts=2026-04-29T20:15:30.105308621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:36304"
level=debug ts=2026-04-29T20:15:31.486500046Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-04-29T20:15:32.815292957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:15:32.820975296Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:15:32.820996454Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:15:32.821018638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:15:32.824512054Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-29T20:15:33.110193325Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:15:34.863710831Z caller=checkpoint.go:372 msg="writing series" size="2.8 MB"
level=debug ts=2026-04-29T20:15:34.863738118Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=2
level=warn ts=2026-04-29T20:15:38.110375807Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:15:38.236742212Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-04-29T20:15:38.528706074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:15:38.535159276Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:15:38.878257198Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:15:41.611671191Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=info ts=2026-04-29T20:15:43.109799179Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:15:43.109830088Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:15:43.109838515Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:15:43.109859639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:15:44.561743367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:15:44.568906013Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:15:44.683243875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:15:44.687192624Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:15:44.986926049Z caller=checkpoint.go:372 msg="writing series" size="487 kB"
ts=2026-04-29T20:15:46.227713976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=warn ts=2026-04-29T20:15:48.110638749Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:15:48.361456266Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=debug ts=2026-04-29T20:15:51.736641538Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=info ts=2026-04-29T20:15:52.582189303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:15:52.590140027Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T20:15:53.110484986Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:15:55.111051125Z caller=checkpoint.go:372 msg="writing series" size="272 kB"
level=debug ts=2026-04-29T20:15:55.111076363Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=warn ts=2026-04-29T20:15:58.109884328Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:15:58.311345327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:15:58.320376372Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T20:15:58.490841036Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-04-29T20:15:58.490864626Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=1
level=debug ts=2026-04-29T20:16:01.861573475Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=warn ts=2026-04-29T20:16:03.110531885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:16:05.236341863Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=debug ts=2026-04-29T20:16:05.447902746Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-29T20:16:05.556278993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:16:05.559490523Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:16:07.542275391Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T20:16:08.11010093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:16:08.611322311Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-04-29T20:16:09.134628026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:16:09.142768636Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:16:11.986335935Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-29T20:16:13.088017977Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:16:13.088026017Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:16:13.110475063Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T20:16:13.110484884Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:16:13.110509865Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:16:13.110522077Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:16:14.22627522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:16:14.230231481Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:16:15.361371429Z caller=checkpoint.go:372 msg="writing series" size="83 kB"
ts=2026-04-29T20:16:16.231154922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=warn ts=2026-04-29T20:16:18.110812243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:16:18.738555732Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-04-29T20:16:21.135639019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:16:21.139786176Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:16:21.139807851Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:16:21.139827311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:16:21.142892397Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:16:22.111756587Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-04-29T20:16:22.111782166Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=7
level=warn ts=2026-04-29T20:16:23.109864674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:16:25.485962071Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=warn ts=2026-04-29T20:16:28.109905551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:16:28.861579816Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
ts=2026-04-29T20:16:30.110814296Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:51812"
level=debug ts=2026-04-29T20:16:32.236529792Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-04-29T20:16:33.110230665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:16:35.611277677Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
level=info ts=2026-04-29T20:16:36.090882897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:16:36.101195746Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-29T20:16:36.892132081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:35180"
level=info ts=2026-04-29T20:16:36.979484532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:16:36.983837343Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-29T20:16:38.110572464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:16:38.673198364Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:16:38.673255379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:16:38.676915034Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-29T20:16:38.98590763Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=info ts=2026-04-29T20:16:41.464141005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:16:41.472098149Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:16:41.472118462Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:16:41.472155702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:16:41.478110363Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:16:42.362179506Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-04-29T20:16:42.362202954Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=6
level=info ts=2026-04-29T20:16:43.11027167Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:16:43.110301541Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:16:43.110310311Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:16:43.110332467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:16:44.040646839Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T20:16:45.736675108Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
ts=2026-04-29T20:16:46.23427145Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=warn ts=2026-04-29T20:16:48.110604745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:16:49.111578496Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-29T20:16:49.111600717Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-04-29T20:16:50.181544095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:16:50.18547684Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-29T20:16:52.487015572Z caller=checkpoint.go:372 msg="writing series" size="1000 kB"
level=warn ts=2026-04-29T20:16:53.110693214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:16:54.436916381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:16:54.443913157Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T20:16:55.390277301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:16:55.394038912Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:16:55.394062208Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:16:55.394090857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:16:55.396979187Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:16:55.861266925Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-04-29T20:16:56.965568483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:16:56.971805394Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T20:16:58.110409646Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:16:59.236606523Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=debug ts=2026-04-29T20:16:59.236628682Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
ts=2026-04-29T20:17:00.113560359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:52056"
level=info ts=2026-04-29T20:17:01.158023169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:17:01.162619195Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:17:02.611508005Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-04-29T20:17:03.110713221Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:17:05.987171907Z caller=checkpoint.go:372 msg="writing series" size="755 kB"
ts=2026-04-29T20:17:06.895261268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:46712"
level=debug ts=2026-04-29T20:17:07.388916147Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:17:08.059133011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:17:08.067890297Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-29T20:17:08.110048233Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:17:09.361739271Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-04-29T20:17:12.223152112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:17:12.227497219Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:17:12.73600377Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=debug ts=2026-04-29T20:17:13.087832102Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:17:13.08889498Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-29T20:17:13.110241411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:17:13.11025385Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:17:13.110278667Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:17:13.110288363Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:17:13.81690089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:17:13.821012766Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:17:16.111612629Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-04-29T20:17:16.111641028Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
ts=2026-04-29T20:17:16.236872423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=warn ts=2026-04-29T20:17:18.110845571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:17:19.485955503Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-04-29T20:17:22.367846755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:17:22.372147195Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-29T20:17:22.861595026Z caller=checkpoint.go:372 msg="writing series" size="347 kB"
level=warn ts=2026-04-29T20:17:23.110198223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:17:23.217519236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:17:23.22499303Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:17:26.236455281Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-04-29T20:17:26.899810165Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-29T20:17:28.110407694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:17:29.611129652Z caller=checkpoint.go:372 msg="writing series" size="627 kB"
level=debug ts=2026-04-29T20:17:32.9868265Z caller=checkpoint.go:372 msg="writing series" size="792 kB"
level=debug ts=2026-04-29T20:17:32.986851727Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=5
level=warn ts=2026-04-29T20:17:33.109835911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:17:35.212279058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:17:35.220759585Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T20:17:36.361153823Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=warn ts=2026-04-29T20:17:38.110253488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:17:39.790129477Z caller=checkpoint.go:372 msg="writing series" size="58 MB"
level=debug ts=2026-04-29T20:17:39.790157415Z caller=checkpoint.go:384 msg="flushing series" totalSize="58 MB" series=2
level=debug ts=2026-04-29T20:17:40.905569294Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:17:42.204241786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:17:42.209533722Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-29T20:17:43.11047725Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:17:43.110505249Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-29T20:17:43.110496574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:17:43.110513278Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:17:43.124254358Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-04-29T20:17:43.125734133Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.015095227s
ts=2026-04-29T20:17:46.240074544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=warn ts=2026-04-29T20:17:48.110831871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:17:51.620667239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:17:51.635645384Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:17:51.635669107Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:17:51.635694118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:17:51.643259914Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=warn ts=2026-04-29T20:17:53.110477771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:17:57.202484186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:17:57.207124496Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-29T20:17:57.262181569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:17:57.266316351Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-29T20:17:58.110814381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:17:59.331467367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:17:59.33906693Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-29T20:18:03.109945275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:18:04.487013125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:18:04.494768702Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T20:18:05.301469643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:18:05.309110985Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T20:18:05.757342817Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:18:06.899391774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:34242"
level=warn ts=2026-04-29T20:18:08.109869633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:18:13.087942379Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:18:13.0881777Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:18:13.109658467Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-29T20:18:13.10981172Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=info ts=2026-04-29T20:18:13.109821153Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-29T20:18:13.10982121Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-29T20:18:13.109837869Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.054µs success=true
level=debug ts=2026-04-29T20:18:13.109844197Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:18:13.1098537Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:18:13.109895324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:18:13.110043379Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=debug ts=2026-04-29T20:18:13.227461898Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-29T20:18:16.242551676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=info ts=2026-04-29T20:18:16.298182174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:18:16.307179389Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T20:18:16.485877477Z caller=checkpoint.go:372 msg="writing series" size="273 kB"
level=warn ts=2026-04-29T20:18:18.110701033Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:18:19.054400566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:18:19.058356327Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-29T20:18:19.860881171Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-04-29T20:18:23.110348626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:18:23.235699259Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-04-29T20:18:26.610966228Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=warn ts=2026-04-29T20:18:28.110886678Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:18:28.417121761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:18:28.425116953Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T20:18:29.985922725Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-04-29T20:18:33.110241777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:18:33.360655474Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=debug ts=2026-04-29T20:18:36.735483684Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
ts=2026-04-29T20:18:36.902137405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:53580"
level=warn ts=2026-04-29T20:18:38.109915477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:18:38.641079011Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:18:40.111117408Z caller=checkpoint.go:372 msg="writing series" size="587 kB"
level=debug ts=2026-04-29T20:18:40.111138728Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=9
level=info ts=2026-04-29T20:18:40.518740363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:18:40.52428919Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-29T20:18:43.110026513Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T20:18:43.110032946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:18:43.11005444Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:18:43.110062871Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T20:18:43.488180493Z caller=checkpoint.go:372 msg="writing series" size="3.9 MB"
level=debug ts=2026-04-29T20:18:43.48820115Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.9 MB" series=1
level=info ts=2026-04-29T20:18:45.698769911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:18:45.703095692Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-04-29T20:18:46.245520671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:18:46.860410576Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=info ts=2026-04-29T20:18:48.040091931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:18:48.047670197Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T20:18:48.109867971Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:18:50.236157192Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=warn ts=2026-04-29T20:18:53.11042113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:18:53.61142222Z caller=checkpoint.go:372 msg="writing series" size="626 kB"
level=info ts=2026-04-29T20:18:54.132391798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:18:54.13638292Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-29T20:18:55.440339297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:18:55.446978897Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:18:56.362622182Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:18:56.364525086Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-29T20:18:56.985483309Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-29T20:18:58.110010956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T20:19:00.123172097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:54824"
level=debug ts=2026-04-29T20:19:00.360392254Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-04-29T20:19:03.110496149Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:19:03.73559266Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-04-29T20:19:07.110348947Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=info ts=2026-04-29T20:19:07.960254426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:19:07.969765372Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:19:07.969788107Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:19:07.969813269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:19:07.97670493Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T20:19:08.110183842Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:19:08.583342693Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:19:08.583407131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="192 B" avg_comp="192 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:19:08.586831888Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-29T20:19:10.485646036Z caller=checkpoint.go:372 msg="writing series" size="268 kB"
level=debug ts=2026-04-29T20:19:10.48568227Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-04-29T20:19:13.088050871Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:19:13.088455025Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:19:13.110776154Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:19:13.110800706Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-29T20:19:13.110791168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:19:13.11080821Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T20:19:13.86056246Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
ts=2026-04-29T20:19:13.929522809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:54028"
ts=2026-04-29T20:19:16.248632862Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=info ts=2026-04-29T20:19:16.437941608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:19:16.441535575Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-29T20:19:17.235519792Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-04-29T20:19:18.110000458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:19:20.154929932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:19:20.159281214Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-29T20:19:20.610830349Z caller=checkpoint.go:372 msg="writing series" size="532 kB"
level=warn ts=2026-04-29T20:19:23.110357027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:19:23.206634855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:19:23.214283264Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T20:19:23.985936453Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=debug ts=2026-04-29T20:19:27.360244979Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=info ts=2026-04-29T20:19:27.746425183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:19:27.75559147Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T20:19:28.110734558Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T20:19:30.12623252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:39280"
level=debug ts=2026-04-29T20:19:30.73601875Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-04-29T20:19:30.776145396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:19:30.780173769Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-04-29T20:19:33.11048596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:19:34.110915049Z caller=checkpoint.go:372 msg="writing series" size="565 kB"
level=debug ts=2026-04-29T20:19:34.110938331Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=7
level=info ts=2026-04-29T20:19:35.186550897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:19:35.19081465Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T20:19:35.947151982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="141 B" avg_comp="141 B" total_uncomp="28 kB" avg_uncomp="28 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-29T20:19:35.947228735Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-04-29T20:19:35.950617184Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-29T20:19:36.247920845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:19:36.251384387Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-29T20:19:36.5472294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="121 B" avg_comp="121 B" total_uncomp="259 kB" avg_uncomp="259 kB" idle=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-04-29T20:19:36.550994351Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19ddaae73c6%3A19ddaae9f05%3A2249b625?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-04-29T20:19:36.847344367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="260 B" avg_comp="260 B" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:19:36.850644999Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-29T20:19:37.147640883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="142 kB" avg_uncomp="142 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:19:37.15148171Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-29T20:19:37.338976408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:19:37.342515844Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-29T20:19:37.447762072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" idle=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:19:37.45079422Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-29T20:19:37.48526767Z caller=checkpoint.go:372 msg="writing series" size="449 kB"
level=info ts=2026-04-29T20:19:37.747704847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:19:37.75148744Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-29T20:19:38.083903785Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-29T20:19:38.109829224Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:19:40.860906199Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=info ts=2026-04-29T20:19:43.110691075Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:19:43.110722712Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-29T20:19:43.110714774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:19:43.11073098Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T20:19:44.235477993Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
ts=2026-04-29T20:19:46.251786255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=info ts=2026-04-29T20:19:46.991230114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:19:46.997927684Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:19:47.611002563Z caller=checkpoint.go:372 msg="writing series" size="761 kB"
level=debug ts=2026-04-29T20:19:47.611026192Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=4
level=info ts=2026-04-29T20:19:47.8713047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:19:47.874256353Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-29T20:19:48.109918058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:19:50.440771884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:19:50.445501758Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-29T20:19:50.985276185Z caller=checkpoint.go:372 msg="writing series" size="262 kB"
level=warn ts=2026-04-29T20:19:53.10998952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:19:53.558947355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:19:53.562061115Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-29T20:19:54.048674753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:19:54.056343744Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:19:54.056374624Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:19:54.056406052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:19:54.06261573Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:19:54.360679024Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-04-29T20:19:55.201270113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="259 kB" avg_uncomp="259 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-04-29T20:19:55.204223225Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19ddaae73c6%3A19ddaae9f05%3A2249b625?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-04-29T20:19:55.540201266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:19:55.543089628Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:19:55.989503845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:19:55.992264319Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-29T20:19:56.641144373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:19:56.64400325Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-29T20:19:57.736805574Z caller=checkpoint.go:372 msg="writing series" size="745 kB"
level=warn ts=2026-04-29T20:19:58.110681842Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:19:59.246842674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:19:59.250138294Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-29T20:20:00.714429203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:20:00.718170534Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-29T20:20:01.110297774Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=debug ts=2026-04-29T20:20:01.110321369Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-04-29T20:20:03.109878379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:20:04.485523414Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=info ts=2026-04-29T20:20:05.090009947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="387 kB" avg_comp="387 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-29T20:20:05.090120234Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-04-29T20:20:05.093860608Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-29T20:20:05.213556191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:20:05.220073457Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T20:20:07.860787611Z caller=checkpoint.go:372 msg="writing series" size="653 B"
level=warn ts=2026-04-29T20:20:08.110621524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:20:10.56373429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:20:10.57479581Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:20:11.236122127Z caller=checkpoint.go:372 msg="writing series" size="591 kB"
level=info ts=2026-04-29T20:20:11.966779921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:20:11.969916705Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-29T20:20:13.087947194Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:20:13.087943114Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:20:13.1099997Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:20:13.110024342Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:20:13.110032864Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:20:13.110063639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:20:14.610634461Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
ts=2026-04-29T20:20:16.254929771Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=info ts=2026-04-29T20:20:17.593501766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:20:17.596665838Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-29T20:20:17.985774876Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=info ts=2026-04-29T20:20:17.989831997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:20:17.992540009Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-29T20:20:18.110737217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:20:18.397472714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="259 kB" avg_uncomp="259 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-04-29T20:20:18.400577511Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19ddaae73c6%3A19ddaae9f05%3A2249b625?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-04-29T20:20:19.29268779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:20:19.296339629Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T20:20:19.66705684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:20:19.670834816Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-29T20:20:21.17619624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:20:21.18522225Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T20:20:21.360995633Z caller=checkpoint.go:372 msg="writing series" size="309 kB"
level=debug ts=2026-04-29T20:20:21.361025093Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-04-29T20:20:22.296167808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:20:22.299383983Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-29T20:20:23.110786875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:20:23.494834967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:20:23.49796788Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:20:24.73551086Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=info ts=2026-04-29T20:20:25.582734718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:20:25.585552196Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-29T20:20:26.046775124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:20:26.049468603Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-29T20:20:28.00712378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:20:28.009499696Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-29T20:20:28.110399275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:20:28.123089719Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-04-29T20:20:28.12310677Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=2
level=debug ts=2026-04-29T20:20:31.485517065Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-04-29T20:20:32.08925881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:20:32.095876457Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T20:20:33.109932241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:20:34.861478586Z caller=checkpoint.go:372 msg="writing series" size="2.9 MB"
level=debug ts=2026-04-29T20:20:34.861504943Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=2
ts=2026-04-29T20:20:36.910510896Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:53946"
level=warn ts=2026-04-29T20:20:38.110262309Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:20:38.187036499Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T20:20:38.235035345Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-04-29T20:20:40.663588164Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:20:41.610459317Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=info ts=2026-04-29T20:20:42.098035211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:20:42.10212853Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-29T20:20:43.110205055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:20:43.110211957Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:20:43.110236704Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:20:43.110244258Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-29T20:20:43.936273208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36170"
level=debug ts=2026-04-29T20:20:44.986272707Z caller=checkpoint.go:372 msg="writing series" size="555 kB"
level=info ts=2026-04-29T20:20:46.188493754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:20:46.195160126Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-29T20:20:46.258334084Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=warn ts=2026-04-29T20:20:48.110689245Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:20:48.360777876Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=info ts=2026-04-29T20:20:51.123320927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:20:51.126709423Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-29T20:20:51.735567269Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=info ts=2026-04-29T20:20:52.538001467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:20:52.54309367Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-29T20:20:53.109889606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:20:54.429930302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:20:54.433039649Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-29T20:20:55.110536135Z caller=checkpoint.go:372 msg="writing series" size="338 kB"
level=debug ts=2026-04-29T20:20:55.110562665Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=info ts=2026-04-29T20:20:55.972579417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:20:55.979076234Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T20:20:58.110303613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:20:58.495885418Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-04-29T20:20:58.495906718Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=1
level=info ts=2026-04-29T20:20:59.83917221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:20:59.842195177Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-29T20:21:00.674024018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:21:00.67706561Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-29T20:21:01.860544976Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-04-29T20:21:02.007932095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:21:02.011727495Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-29T20:21:03.109952418Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:21:04.145179584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:21:04.153973468Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T20:21:04.367492254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:21:04.37105156Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:21:05.235999361Z caller=checkpoint.go:372 msg="writing series" size="935 B"
ts=2026-04-29T20:21:06.913581592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:50078"
level=debug ts=2026-04-29T20:21:06.942317643Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T20:21:08.110216909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:21:08.610371869Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-04-29T20:21:09.042194994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="259 kB" avg_uncomp="259 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-04-29T20:21:09.045227809Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19ddaae73c6%3A19ddaae9f05%3A2249b625?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-04-29T20:21:11.985876579Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-29T20:21:13.088044105Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:21:13.08857324Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=warn ts=2026-04-29T20:21:13.109892227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:21:13.109896191Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:21:13.109923854Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:21:13.109933025Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T20:21:15.360647224Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
ts=2026-04-29T20:21:16.26126764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=info ts=2026-04-29T20:21:16.823145503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:21:16.826984704Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-29T20:21:18.080478884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:21:18.083472958Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-29T20:21:18.110881964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:21:18.447653416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:21:18.451270984Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-29T20:21:18.735249884Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-04-29T20:21:20.037871976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:21:20.046578741Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T20:21:20.846379487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:21:20.850230388Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:21:22.111722448Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-04-29T20:21:22.111745019Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=7
level=warn ts=2026-04-29T20:21:23.109874463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:21:25.486048707Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=warn ts=2026-04-29T20:21:28.109980345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:21:28.255553812Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T20:21:28.860409248Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
level=info ts=2026-04-29T20:21:31.209356619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:21:31.21237582Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-29T20:21:32.235350779Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-04-29T20:21:33.109877046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:21:35.610927681Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=warn ts=2026-04-29T20:21:38.110761125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:21:38.985198625Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=debug ts=2026-04-29T20:21:40.895576847Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:21:41.075863515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:21:41.080114322Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-29T20:21:42.361969234Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-04-29T20:21:42.36199232Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=6
level=info ts=2026-04-29T20:21:43.110141715Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T20:21:43.110145354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:21:43.110174214Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:21:43.110192434Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:21:44.260327873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:21:44.267206188Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-29T20:21:45.619655639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:21:45.626944394Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T20:21:45.735458254Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-04-29T20:21:45.856785931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:21:45.863390417Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:21:46.184598455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:21:46.188556378Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:21:46.188576295Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:21:46.188598522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:21:46.191726644Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-04-29T20:21:46.263873145Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=info ts=2026-04-29T20:21:46.622232086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:21:46.624748558Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-29T20:21:47.835311996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="944 B" avg_comp="944 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:21:47.838279512Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-29T20:21:48.114744479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:21:49.111762662Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-29T20:21:49.111783759Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-04-29T20:21:49.676274358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:21:49.67937429Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-29T20:21:51.907134378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:21:51.910039926Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-29T20:21:52.485832026Z caller=checkpoint.go:372 msg="writing series" size="1000 kB"
level=warn ts=2026-04-29T20:21:53.110452967Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:21:55.824687272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:21:55.829604823Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:21:55.860692071Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-04-29T20:21:58.110363494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:21:59.235472441Z caller=checkpoint.go:372 msg="writing series" size="236 kB"
level=debug ts=2026-04-29T20:21:59.23549469Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=info ts=2026-04-29T20:22:01.655283104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:22:01.659213174Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:22:02.610364242Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-29T20:22:02.732762041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:22:02.739343796Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T20:22:03.110699606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:22:05.466730156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:22:05.469985241Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-29T20:22:05.986074068Z caller=checkpoint.go:372 msg="writing series" size="779 kB"
level=info ts=2026-04-29T20:22:06.537407737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:22:06.54035541Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-29T20:22:06.686850622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="259 kB" avg_uncomp="259 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-04-29T20:22:06.689883146Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19ddaae73c6%3A19ddaae9f05%3A2249b625?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-04-29T20:22:06.919558062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:36494"
level=warn ts=2026-04-29T20:22:08.110863452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:22:08.28475576Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:22:09.360222991Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-04-29T20:22:11.903426181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:22:11.906876184Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:22:12.735279192Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-04-29T20:22:13.088110021Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:22:13.088678648Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:22:13.110028043Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:22:13.110051502Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:22:13.110062566Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:22:13.110072862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T20:22:13.943853038Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:45730"
level=info ts=2026-04-29T20:22:15.938195339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:22:15.941106494Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-29T20:22:16.111089389Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-04-29T20:22:16.111113867Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
ts=2026-04-29T20:22:16.266248257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=warn ts=2026-04-29T20:22:18.110494777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:22:19.485956983Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-04-29T20:22:20.800634255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:22:20.804602206Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-29T20:22:22.861333569Z caller=checkpoint.go:372 msg="writing series" size="347 kB"
level=warn ts=2026-04-29T20:22:23.110306599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:22:23.742726324Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T20:22:26.23562412Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-04-29T20:22:28.110852817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:22:28.602237267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:22:28.610042309Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-29T20:22:28.634685126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:22:28.637835774Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T20:22:29.355156439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:22:29.35880401Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-29T20:22:29.610706801Z caller=checkpoint.go:372 msg="writing series" size="627 kB"
level=info ts=2026-04-29T20:22:30.074106281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:22:30.080924005Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T20:22:31.117026754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:22:31.120152742Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-29T20:22:31.267402404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:22:31.274175397Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T20:22:32.986316859Z caller=checkpoint.go:372 msg="writing series" size="934 kB"
level=debug ts=2026-04-29T20:22:32.986339769Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=5
level=warn ts=2026-04-29T20:22:33.110617249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:22:36.360607334Z caller=checkpoint.go:372 msg="writing series" size="206 kB"
level=warn ts=2026-04-29T20:22:38.110418704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:22:39.680719229Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:22:39.788013519Z caller=checkpoint.go:372 msg="writing series" size="58 MB"
level=debug ts=2026-04-29T20:22:39.788048386Z caller=checkpoint.go:384 msg="flushing series" totalSize="58 MB" series=2
level=info ts=2026-04-29T20:22:42.088403601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:22:42.094040788Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:22:42.094065318Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:22:42.094087069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:22:42.097646423Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-29T20:22:43.109885564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:22:43.109877411Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:22:43.109911395Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:22:43.109919575Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:22:43.122556664Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000011.tmp new=/var/loki/wal/checkpoint.000011
level=info ts=2026-04-29T20:22:43.124386354Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.014410642s
level=info ts=2026-04-29T20:22:44.262127315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:22:44.265044595Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-29T20:22:45.44071797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:22:45.444164353Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-04-29T20:22:46.269453458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=info ts=2026-04-29T20:22:47.837468923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:22:47.840381746Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-29T20:22:48.109885592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:22:49.864958532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:22:49.871954093Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:22:49.871977148Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:22:49.872014606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:22:49.87857703Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T20:22:52.825072767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:22:52.829135772Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-29T20:22:53.110343557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:22:53.892198378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:22:53.895704252Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-29T20:22:54.310958588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:22:54.315100171Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-29T20:22:55.992813372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:22:55.99564496Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-29T20:22:56.951577499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:22:56.954910946Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-29T20:22:58.110236479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T20:23:00.142383913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:41932"
level=warn ts=2026-04-29T20:23:03.109849961Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:23:04.15725889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:23:04.161582662Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-29T20:23:05.318411519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="259 kB" avg_uncomp="259 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-04-29T20:23:05.321603481Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19ddaae73c6%3A19ddaae9f05%3A2249b625?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-04-29T20:23:05.938262453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:23:05.946114372Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:23:06.995325231Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T20:23:08.110160518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:23:13.08804117Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:23:13.08852405Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:23:13.109756407Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-29T20:23:13.109819934Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T20:23:13.109836505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:23:13.109844756Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:23:13.109855452Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:23:13.10993908Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000012
level=info ts=2026-04-29T20:23:13.109947482Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-29T20:23:13.109967884Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.376µs success=true
level=debug ts=2026-04-29T20:23:13.110180056Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
ts=2026-04-29T20:23:13.948282619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:35652"
level=info ts=2026-04-29T20:23:14.326189323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:23:14.331172115Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-29T20:23:14.775762109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:23:14.779195675Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-29T20:23:14.845910166Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:23:14.847733266Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T20:23:16.272858129Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=info ts=2026-04-29T20:23:16.460936744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:23:16.464259774Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:23:16.485691949Z caller=checkpoint.go:372 msg="writing series" size="273 kB"
level=warn ts=2026-04-29T20:23:18.11042244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:23:19.860407633Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=info ts=2026-04-29T20:23:20.286248334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:23:20.293231617Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T20:23:21.108554342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:23:21.114984886Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-29T20:23:23.008561502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:23:23.012910183Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:23:23.012931133Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:23:23.012952571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:23:23.016096265Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-29T20:23:23.110367133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:23:23.235723881Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-04-29T20:23:25.533494829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:23:25.536139507Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-29T20:23:26.61039315Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=warn ts=2026-04-29T20:23:28.110871766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:23:28.627090429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:23:28.634211684Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:23:29.254008027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:23:29.257109966Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-29T20:23:29.98578205Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
ts=2026-04-29T20:23:30.145964334Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:56184"
level=info ts=2026-04-29T20:23:30.949490019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:23:30.9522626Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-29T20:23:33.110450112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:23:33.361133002Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=debug ts=2026-04-29T20:23:35.239910301Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:23:35.239910769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="134 B" avg_comp="134 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" idle=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:23:35.244720759Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=debug ts=2026-04-29T20:23:36.735787378Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-04-29T20:23:38.110632153Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:23:39.96812845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:23:39.971430142Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-29T20:23:40.110950282Z caller=checkpoint.go:372 msg="writing series" size="587 kB"
level=debug ts=2026-04-29T20:23:40.110971481Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=9
level=info ts=2026-04-29T20:23:41.932755576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:23:41.936734105Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-29T20:23:43.110296101Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:23:43.110325816Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:23:43.110335911Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:23:43.110349537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:23:43.49081959Z caller=checkpoint.go:372 msg="writing series" size="4.0 MB"
level=debug ts=2026-04-29T20:23:43.490839489Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.0 MB" series=1
ts=2026-04-29T20:23:43.951528286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:58908"
level=info ts=2026-04-29T20:23:45.937484437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:23:45.945497016Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T20:23:46.272559195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-04-29T20:23:46.275266713Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=error ts=2026-04-29T20:23:46.275423741Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-29T20:23:46.86020867Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-04-29T20:23:47.012827847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:23:47.016240422Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:23:47.016260096Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:23:47.016279082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:23:47.01928712Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-04-29T20:23:48.110393518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:23:50.236127191Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=warn ts=2026-04-29T20:23:53.110393231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:23:53.34779892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:23:53.35132929Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=debug ts=2026-04-29T20:23:53.611641497Z caller=checkpoint.go:372 msg="writing series" size="434 kB"
level=info ts=2026-04-29T20:23:53.960098943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:23:53.962921798Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-29T20:23:56.98619642Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-29T20:23:58.110213002Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:23:58.59825216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:23:58.602614882Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-29T20:23:59.612664058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="259 kB" avg_uncomp="259 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-04-29T20:23:59.61575522Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19ddaae73c6%3A19ddaae9f05%3A2249b625?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-04-29T20:24:00.360527768Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-04-29T20:24:02.695483045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:24:02.699297491Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-29T20:24:02.899874366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:24:02.907848613Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:24:02.907868105Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:24:02.907892812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:24:02.91498932Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=warn ts=2026-04-29T20:24:03.109826587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:24:03.138927697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:24:03.141711309Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-29T20:24:03.735617455Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-04-29T20:24:05.56549022Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T20:24:06.47646745Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:24:07.111290889Z caller=checkpoint.go:372 msg="writing series" size="154 kB"
level=info ts=2026-04-29T20:24:07.161176257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:24:07.164332658Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-29T20:24:08.109969036Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:24:09.235074178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:24:09.242512183Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:24:10.485303357Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=info ts=2026-04-29T20:24:11.263475331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:24:11.266371948Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-29T20:24:13.087746746Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:24:13.088785694Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-29T20:24:13.110188705Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:24:13.110214634Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:24:13.11022238Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:24:13.110250043Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:24:13.546656913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:24:13.550288937Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:24:13.861266759Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
level=info ts=2026-04-29T20:24:14.26528937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:24:14.269185713Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-29T20:24:15.065282296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:24:15.06794442Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:24:15.161396334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:24:15.168793197Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
ts=2026-04-29T20:24:16.278193479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=info ts=2026-04-29T20:24:17.06242283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:24:17.06494085Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-29T20:24:17.235677176Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=debug ts=2026-04-29T20:24:17.235699871Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-04-29T20:24:17.899614085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:24:17.902195811Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-04-29T20:24:18.110360714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:24:20.610786772Z caller=checkpoint.go:372 msg="writing series" size="532 kB"
level=warn ts=2026-04-29T20:24:23.110820935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:24:23.588483617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:24:23.597219959Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T20:24:23.985728352Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=debug ts=2026-04-29T20:24:27.361096348Z caller=checkpoint.go:372 msg="writing series" size="6.5 kB"
level=warn ts=2026-04-29T20:24:28.110823399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:24:28.277207424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:24:28.282221557Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-29T20:24:29.389187399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:24:29.392687187Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-29T20:24:29.645915707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:24:29.652789081Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:24:30.736208647Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-29T20:24:33.110477283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:24:33.528467424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:24:33.531759286Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:24:34.110821901Z caller=checkpoint.go:372 msg="writing series" size="473 kB"
level=debug ts=2026-04-29T20:24:34.110844072Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-04-29T20:24:36.170854905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:24:36.175905931Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-29T20:24:37.486093939Z caller=checkpoint.go:372 msg="writing series" size="561 kB"
level=debug ts=2026-04-29T20:24:37.824823206Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T20:24:38.110630188Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:24:40.860980154Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=info ts=2026-04-29T20:24:42.510027411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:24:42.516981001Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T20:24:42.917091657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:24:42.921193685Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-29T20:24:43.110417095Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:24:43.110465173Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-29T20:24:43.110458245Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:24:43.110475328Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-29T20:24:43.955807937Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:48494"
level=debug ts=2026-04-29T20:24:44.235881374Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
level=info ts=2026-04-29T20:24:46.203390073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="259 kB" avg_uncomp="259 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-04-29T20:24:46.206584075Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19ddaae73c6%3A19ddaae9f05%3A2249b625?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-04-29T20:24:46.280774935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T20:24:47.610819414Z caller=checkpoint.go:372 msg="writing series" size="761 kB"
level=debug ts=2026-04-29T20:24:47.610843485Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=4
level=warn ts=2026-04-29T20:24:48.10984743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:24:50.986027449Z caller=checkpoint.go:372 msg="writing series" size="280 kB"
level=info ts=2026-04-29T20:24:51.809483323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:24:51.812895969Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-29T20:24:53.109862566Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:24:54.361039487Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-04-29T20:24:57.736356272Z caller=checkpoint.go:372 msg="writing series" size="745 kB"
level=warn ts=2026-04-29T20:24:58.110176055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:24:58.69103211Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T20:25:01.110721229Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=debug ts=2026-04-29T20:25:01.110745438Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-04-29T20:25:02.44734708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:25:02.452807506Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-29T20:25:03.110596252Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:25:03.688263097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:25:03.691098499Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-29T20:25:04.450127797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:25:04.452725927Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-29T20:25:04.486090301Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
level=info ts=2026-04-29T20:25:05.121953712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:25:05.125373623Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-29T20:25:06.363016051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:25:06.372850568Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-04-29T20:25:06.931928965Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:40908"
level=info ts=2026-04-29T20:25:07.657488851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:25:07.663813761Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:25:07.860368805Z caller=checkpoint.go:372 msg="writing series" size="653 B"
level=warn ts=2026-04-29T20:25:08.110322484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:25:11.236074884Z caller=checkpoint.go:372 msg="writing series" size="591 kB"
level=info ts=2026-04-29T20:25:11.659017452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:25:11.661521753Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-29T20:25:12.717709375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:25:12.724079675Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T20:25:13.087894886Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:25:13.088101947Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:25:13.110469952Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:25:13.11049455Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-29T20:25:13.110488326Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:25:13.110502184Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T20:25:13.763911076Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:25:14.011648034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:25:14.014170971Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=debug ts=2026-04-29T20:25:14.610929886Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
ts=2026-04-29T20:25:16.283510173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=info ts=2026-04-29T20:25:16.800199551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:25:16.806524435Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:25:17.203715886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:25:17.206815416Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-29T20:25:17.986152502Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=warn ts=2026-04-29T20:25:18.110490453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:25:21.360815984Z caller=checkpoint.go:372 msg="writing series" size="343 kB"
level=debug ts=2026-04-29T20:25:21.360839912Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=warn ts=2026-04-29T20:25:23.110168073Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:25:24.735451299Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=info ts=2026-04-29T20:25:27.377487542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:25:27.386169839Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-29T20:25:27.78823205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:25:27.791830507Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-29T20:25:27.874901482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:25:27.877985677Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-29T20:25:28.110411142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:25:28.124816798Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-04-29T20:25:28.124836147Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=2
level=info ts=2026-04-29T20:25:30.607204772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="259 kB" avg_uncomp="259 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-04-29T20:25:30.610294853Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19ddaae73c6%3A19ddaae9f05%3A2249b625?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-04-29T20:25:31.485577127Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-04-29T20:25:32.644942972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:25:32.648793722Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-29T20:25:33.110482038Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:25:34.862482175Z caller=checkpoint.go:372 msg="writing series" size="2.8 MB"
level=debug ts=2026-04-29T20:25:34.862505246Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=2
level=warn ts=2026-04-29T20:25:38.11026818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:25:38.235567485Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-04-29T20:25:38.647459585Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:25:38.882241716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:25:38.890892968Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T20:25:41.610987385Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=debug ts=2026-04-29T20:25:42.520958478Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-29T20:25:43.109783062Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:25:43.109822292Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:25:43.109833442Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:25:43.109845599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:25:43.123014223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:25:43.127052581Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-04-29T20:25:43.96197188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36468"
level=info ts=2026-04-29T20:25:44.862542074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:25:44.865549256Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-04-29T20:25:44.865571578Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:25:44.865591522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:25:44.867843993Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-29T20:25:44.985836116Z caller=checkpoint.go:372 msg="writing series" size="448 kB"
ts=2026-04-29T20:25:46.28625999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=warn ts=2026-04-29T20:25:48.110797645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:25:48.361166338Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=debug ts=2026-04-29T20:25:51.735331253Z caller=checkpoint.go:372 msg="writing series" size="286 kB"
level=warn ts=2026-04-29T20:25:53.110281343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:25:55.110510184Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=debug ts=2026-04-29T20:25:55.110533265Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-04-29T20:25:55.488482762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:25:55.493826431Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:25:55.493852334Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:25:55.493874004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:25:55.496519414Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:25:55.721189784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:25:55.724458134Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:25:55.724485335Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:25:55.794717465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:25:55.798360289Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-29T20:25:57.272355555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:25:57.275864165Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-29T20:25:57.614891465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:25:57.618704108Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-29T20:25:58.11061117Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:25:58.496535197Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-04-29T20:25:58.496556493Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=1
level=debug ts=2026-04-29T20:26:01.860822573Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=info ts=2026-04-29T20:26:02.064404995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:26:02.067798557Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:26:02.067819253Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:26:02.067840872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:26:02.070097859Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-29T20:26:02.297159402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:26:02.304777025Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T20:26:03.110743377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:26:03.325271256Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:26:04.381704582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:26:04.385270318Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-29T20:26:05.235386596Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=info ts=2026-04-29T20:26:06.556397757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:26:06.562935499Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-29T20:26:06.936826289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:37962"
level=warn ts=2026-04-29T20:26:08.110143953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:26:08.611157348Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-04-29T20:26:09.707820245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:26:09.711926105Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:26:10.355665188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:26:10.362448346Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:26:10.362473008Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:26:10.362498542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:26:10.369243492Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:26:10.989037286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:26:10.992378484Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:26:10.992403481Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:26:10.99245089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:26:10.994825099Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-29T20:26:11.322908145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:26:11.325790298Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:26:11.325811694Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:26:11.325831587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:26:11.328059652Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-29T20:26:11.986083777Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-29T20:26:12.058085544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:26:12.061163207Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-29T20:26:12.854369878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:26:12.856859433Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-29T20:26:13.088016409Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:26:13.088022776Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:26:13.110428697Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:26:13.110473404Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-29T20:26:13.110471742Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:26:13.110481219Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:26:13.71137401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:26:13.713800559Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-29T20:26:14.812123239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:26:14.815378147Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:26:15.361144003Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
ts=2026-04-29T20:26:16.28903242Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=info ts=2026-04-29T20:26:17.799647527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="259 kB" avg_uncomp="259 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-04-29T20:26:17.802536058Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19ddaae73c6%3A19ddaae9f05%3A2249b625?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-04-29T20:26:17.80255751Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:26:17.802577149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="122 B" avg_comp="122 B" total_uncomp="46 kB" avg_uncomp="46 kB" idle=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:26:17.804805451Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-04-29T20:26:18.110288704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:26:18.736054496Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-04-29T20:26:20.875165758Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-29T20:26:20.954455056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:26:20.957226253Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-29T20:26:22.112060886Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-04-29T20:26:22.112084487Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=7
level=info ts=2026-04-29T20:26:22.345715847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:26:22.348624901Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-29T20:26:23.110176288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:26:23.934788159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:26:23.937632004Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:26:23.93765097Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:26:23.937670544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:26:23.940016353Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-29T20:26:24.657240077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:26:24.661273693Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:26:25.48595718Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=info ts=2026-04-29T20:26:26.180967395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:26:26.183868602Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-29T20:26:27.486118627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:26:27.489760035Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T20:26:27.798565394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:26:27.80704697Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=warn ts=2026-04-29T20:26:28.110750535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:26:28.860846098Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
level=info ts=2026-04-29T20:26:29.099612186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:26:29.106079029Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T20:26:31.728978293Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:26:32.235396304Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-04-29T20:26:32.61050528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:26:32.613356875Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-29T20:26:33.109857811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:26:34.572502332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:26:34.575070927Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-29T20:26:35.611242142Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=info ts=2026-04-29T20:26:37.429480315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:26:37.432277178Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-29T20:26:37.58278784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:26:37.585984671Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-04-29T20:26:38.110396647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:26:38.985853804Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=debug ts=2026-04-29T20:26:42.36250845Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-04-29T20:26:42.362529053Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=6
level=info ts=2026-04-29T20:26:43.110746732Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:26:43.110772986Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:26:43.110780603Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:26:43.110786882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T20:26:43.967605432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:36264"
level=debug ts=2026-04-29T20:26:45.735592707Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
ts=2026-04-29T20:26:46.292415939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=info ts=2026-04-29T20:26:48.07368656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="944 B" avg_comp="944 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:26:48.079156719Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-29T20:26:48.110282991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:26:49.111167592Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-04-29T20:26:49.111190323Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-04-29T20:26:50.78963457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:26:50.793498859Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:26:50.793516726Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:26:50.793536991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:26:50.79627853Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-29T20:26:51.55848315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:26:51.561197409Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-29T20:26:52.486038704Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-29T20:26:52.48606037Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=warn ts=2026-04-29T20:26:53.109914907Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:26:53.965279729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:26:53.972955625Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:26:53.972976589Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:26:53.972999808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:26:53.979918315Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:26:55.860733464Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-04-29T20:26:57.19999133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:26:57.207050607Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T20:26:58.110108805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:26:59.235781324Z caller=checkpoint.go:372 msg="writing series" size="223 kB"
level=debug ts=2026-04-29T20:26:59.255946691Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:27:00.493465091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:27:00.498011957Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-29T20:27:01.825725422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:27:01.829117639Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-29T20:27:02.611378563Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-29T20:27:02.758663259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:27:02.766464857Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:27:02.837636522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:27:02.840159721Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-29T20:27:03.021780934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:27:03.02440486Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-29T20:27:03.110596042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:27:05.986084326Z caller=checkpoint.go:372 msg="writing series" size="817 kB"
level=debug ts=2026-04-29T20:27:05.986107262Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
ts=2026-04-29T20:27:06.941652161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:44186"
level=debug ts=2026-04-29T20:27:07.510890869Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:27:07.512670604Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-29T20:27:08.107379129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=warn ts=2026-04-29T20:27:08.109830894Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-04-29T20:27:08.110046457Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-29T20:27:08.991028605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:27:08.994344895Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-29T20:27:09.360518801Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-04-29T20:27:10.288929948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:27:10.296709339Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:27:12.736230825Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-04-29T20:27:13.087903514Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:27:13.088363494Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:27:13.110734226Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:27:13.110758928Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:27:13.110766505Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:27:13.110770008Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:27:13.484573349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:27:13.488355525Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:27:16.110673478Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
ts=2026-04-29T20:27:16.29501861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=info ts=2026-04-29T20:27:17.098611295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:27:17.101473852Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-29T20:27:17.12954483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:27:17.139051763Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T20:27:17.350324119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:27:17.354235809Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-29T20:27:18.110612317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:27:18.206894604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:27:18.21123003Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-29T20:27:19.485881333Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-04-29T20:27:22.860999727Z caller=checkpoint.go:372 msg="writing series" size="347 kB"
level=warn ts=2026-04-29T20:27:23.109840192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:27:25.182305675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:27:25.185532127Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-29T20:27:26.235468381Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-04-29T20:27:26.673663084Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T20:27:28.110898588Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:27:29.61142002Z caller=checkpoint.go:372 msg="writing series" size="627 kB"
level=debug ts=2026-04-29T20:27:29.611460847Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=7
ts=2026-04-29T20:27:30.164646603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:35154"
level=info ts=2026-04-29T20:27:30.737490979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:27:30.741102256Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-29T20:27:32.986237486Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-29T20:27:32.986258997Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=warn ts=2026-04-29T20:27:33.110269266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:27:33.808816903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:27:33.812034514Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-29T20:27:34.007602513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:27:34.01082648Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-29T20:27:36.361127052Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
level=warn ts=2026-04-29T20:27:38.11011011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:27:39.790400806Z caller=checkpoint.go:372 msg="writing series" size="58 MB"
level=debug ts=2026-04-29T20:27:39.790428509Z caller=checkpoint.go:384 msg="flushing series" totalSize="59 MB" series=2
level=info ts=2026-04-29T20:27:40.933488122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:27:40.941171837Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-29T20:27:43.110775801Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:27:43.110805879Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:27:43.110814832Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:27:43.110853068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:27:43.134144548Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000012.tmp new=/var/loki/wal/checkpoint.000012
level=info ts=2026-04-29T20:27:43.135403996Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.025289342s
ts=2026-04-29T20:27:43.973351161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:34180"
level=info ts=2026-04-29T20:27:44.543866641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:27:44.547039563Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
ts=2026-04-29T20:27:46.297709488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T20:27:46.831313872Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-29T20:27:48.110579444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:27:48.127745628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:27:48.134800123Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T20:27:49.353211333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:27:49.357329456Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-29T20:27:52.779019132Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T20:27:53.110299754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:27:53.217595425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:27:53.221842381Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-29T20:27:53.396898687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:27:53.404790238Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:27:57.33309805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:27:57.336362299Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-29T20:27:57.698479808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:27:57.701572658Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-04-29T20:27:58.110636624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T20:28:00.168053033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:53898"
level=info ts=2026-04-29T20:28:01.794586458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:28:01.799036953Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-29T20:28:02.983552112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:28:02.987677907Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-29T20:28:03.110184708Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:28:03.1222808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:28:03.126278138Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:28:03.12629974Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:28:03.12632225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:28:03.129529802Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T20:28:05.693282061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:28:05.700852414Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T20:28:06.973380536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:28:06.976459919Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-29T20:28:07.378650304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:28:07.3816479Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-29T20:28:08.110750827Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:28:11.481773455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:28:11.484782773Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:28:13.087897404Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:28:13.088430455Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:28:13.109671609Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-29T20:28:13.110052175Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000013
level=info ts=2026-04-29T20:28:13.110064283Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-29T20:28:13.110083283Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.307µs success=true
level=info ts=2026-04-29T20:28:13.110091991Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T20:28:13.110105895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:28:13.110122127Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:28:13.110131189Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T20:28:13.110313053Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
ts=2026-04-29T20:28:16.300514958Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=info ts=2026-04-29T20:28:16.321889914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:28:16.325677572Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:28:16.486392176Z caller=checkpoint.go:372 msg="writing series" size="294 kB"
level=info ts=2026-04-29T20:28:17.022628636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:28:17.025261866Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-29T20:28:17.646485682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:28:17.649535241Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-04-29T20:28:18.110067116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:28:19.860373998Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=debug ts=2026-04-29T20:28:21.277809555Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T20:28:23.109906767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:28:23.236265877Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-04-29T20:28:26.611105395Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=info ts=2026-04-29T20:28:27.67193573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:28:27.680567918Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-29T20:28:28.109863783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:28:29.986424199Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-04-29T20:28:31.468772678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:28:31.472289257Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-29T20:28:32.37808584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:28:32.382360467Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-29T20:28:33.110485723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:28:33.361077468Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=debug ts=2026-04-29T20:28:36.735859002Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-04-29T20:28:38.109913966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:28:38.436952698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:28:38.440806184Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-29T20:28:40.016654316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T20:28:40.111835145Z caller=checkpoint.go:372 msg="writing series" size="619 kB"
level=debug ts=2026-04-29T20:28:40.11185926Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=9
level=info ts=2026-04-29T20:28:43.110240824Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T20:28:43.11026186Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:28:43.110274564Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:28:43.110284226Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T20:28:43.487748951Z caller=checkpoint.go:372 msg="writing series" size="3.9 MB"
level=debug ts=2026-04-29T20:28:43.487770007Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.9 MB" series=1
level=info ts=2026-04-29T20:28:44.250335822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:28:44.254124404Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:28:44.254151097Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:28:44.254179899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:28:44.257378845Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-29T20:28:45.101945345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:28:45.104517659Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-29T20:28:45.177553577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:28:45.179956279Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-29T20:28:46.102669249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:28:46.10915957Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-29T20:28:46.303716483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:28:46.860357727Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=debug ts=2026-04-29T20:28:47.789404223Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:28:47.78949393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:28:47.79631998Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-29T20:28:48.110794061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:28:48.481481423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="889 B" avg_comp="889 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:28:48.48405103Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-29T20:28:50.047499289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:28:50.055070477Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T20:28:50.235974732Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=info ts=2026-04-29T20:28:52.292118354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:28:52.295608047Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-29T20:28:53.110200868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:28:53.611188467Z caller=checkpoint.go:372 msg="writing series" size="470 kB"
level=info ts=2026-04-29T20:28:54.406981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:28:54.40989351Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:28:55.06175347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:28:55.065674271Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:28:56.98572729Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-29T20:28:58.110180826Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:28:59.448817137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:28:59.453133226Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-29T20:29:00.361087503Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=info ts=2026-04-29T20:29:00.469351056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:29:00.476863776Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T20:29:01.778931333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:29:01.783054654Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-29T20:29:02.730100358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:29:02.736620714Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-29T20:29:03.110516408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:29:03.735525275Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-04-29T20:29:06.721620227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:29:06.724923805Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-29T20:29:07.111061468Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
level=warn ts=2026-04-29T20:29:08.110195061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:29:09.836754016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:29:09.839589038Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-29T20:29:10.39865063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:29:10.406126552Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:29:10.485460888Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=info ts=2026-04-29T20:29:12.987713222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:29:12.991001644Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-29T20:29:13.088061122Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:29:13.088481581Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:29:13.109791787Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:29:13.10982212Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:29:13.10982968Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:29:13.1098307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:29:13.860871959Z caller=checkpoint.go:372 msg="writing series" size="305 kB"
level=debug ts=2026-04-29T20:29:13.860893748Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
ts=2026-04-29T20:29:16.306252631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:29:16.731944993Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:29:17.2353378Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=warn ts=2026-04-29T20:29:18.110044019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:29:20.610869364Z caller=checkpoint.go:372 msg="writing series" size="532 kB"
level=info ts=2026-04-29T20:29:21.522481239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:29:21.527559687Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-29T20:29:23.110818691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:29:23.986080834Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
level=info ts=2026-04-29T20:29:25.17228802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:29:25.178940703Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-29T20:29:25.784838785Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T20:29:27.36050314Z caller=checkpoint.go:372 msg="writing series" size="6.5 kB"
level=warn ts=2026-04-29T20:29:28.110828789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:29:29.548648704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:29:29.553375528Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-29T20:29:29.939327355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:29:29.942885584Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:29:30.736026326Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-04-29T20:29:31.297957513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:29:31.300696615Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-29T20:29:31.53641947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:29:31.53961192Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-29T20:29:32.00886844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:29:32.011620745Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-29T20:29:33.110289464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:29:34.111583302Z caller=checkpoint.go:372 msg="writing series" size="574 kB"
level=debug ts=2026-04-29T20:29:34.111606077Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-04-29T20:29:36.6034705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:29:36.607386851Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:29:36.969426267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:29:36.976741621Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:29:36.976763442Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:29:36.97679763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:29:36.983318529Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:29:37.486177547Z caller=checkpoint.go:372 msg="writing series" size="462 kB"
level=warn ts=2026-04-29T20:29:38.110471011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:29:40.860811135Z caller=checkpoint.go:372 msg="writing series" size="245 kB"
level=info ts=2026-04-29T20:29:43.109855785Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:29:43.109883865Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:29:43.109891921Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:29:43.109968051Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:29:43.716016026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:29:43.719566711Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-04-29T20:29:43.982844666Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:45808"
level=debug ts=2026-04-29T20:29:44.235995806Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
ts=2026-04-29T20:29:46.309594711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=info ts=2026-04-29T20:29:47.339632057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:29:47.343581856Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T20:29:47.461844811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:29:47.469513991Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T20:29:47.611286577Z caller=checkpoint.go:372 msg="writing series" size="761 kB"
level=debug ts=2026-04-29T20:29:47.611310129Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=4
level=warn ts=2026-04-29T20:29:48.110647068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:29:48.220971181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:29:48.224310276Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=debug ts=2026-04-29T20:29:49.919249724Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:29:50.985402765Z caller=checkpoint.go:372 msg="writing series" size="280 kB"
level=info ts=2026-04-29T20:29:52.629195161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:29:52.637547468Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=warn ts=2026-04-29T20:29:53.110018717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:29:54.001231065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:29:54.008977648Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:29:54.361062517Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-04-29T20:29:56.71125922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:29:56.713852096Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-29T20:29:57.736751081Z caller=checkpoint.go:372 msg="writing series" size="745 kB"
level=info ts=2026-04-29T20:29:57.797283259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:29:57.800572842Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-29T20:29:57.946010474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:29:57.952516211Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-29T20:29:58.110214044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:30:00.813210936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:30:00.816401427Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-29T20:30:01.111205537Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=debug ts=2026-04-29T20:30:01.111226079Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=warn ts=2026-04-29T20:30:03.110419966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:30:03.68377804Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T20:30:04.48602962Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
ts=2026-04-29T20:30:06.952467556Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:38794"
level=info ts=2026-04-29T20:30:07.270879511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:30:07.274109683Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-29T20:30:07.860747852Z caller=checkpoint.go:372 msg="writing series" size="653 B"
level=info ts=2026-04-29T20:30:07.97519262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:30:07.978671645Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:30:07.978692252Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:30:07.978714725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:30:07.981744755Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-29T20:30:08.110221411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:30:11.236204632Z caller=checkpoint.go:372 msg="writing series" size="591 kB"
level=info ts=2026-04-29T20:30:13.087905356Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:30:13.088266662Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=debug ts=2026-04-29T20:30:13.089572302Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-29T20:30:13.08961728Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974993 n=1
level=debug ts=2026-04-29T20:30:13.089658791Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974993 n=1
level=info ts=2026-04-29T20:30:13.110605767Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:30:13.110626836Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:30:13.110634797Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:30:13.11065301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:30:14.610636575Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
ts=2026-04-29T20:30:16.312771912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T20:30:17.985732833Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=warn ts=2026-04-29T20:30:18.11021694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:30:18.365022146Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:30:20.689851349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:30:20.694136972Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-29T20:30:21.361267807Z caller=checkpoint.go:372 msg="writing series" size="343 kB"
level=debug ts=2026-04-29T20:30:21.361296307Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=info ts=2026-04-29T20:30:22.678481294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:30:22.683389721Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-29T20:30:23.110418711Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:30:23.7346764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:30:23.74234352Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-29T20:30:24.736071141Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-04-29T20:30:25.078947133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:30:25.082216145Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-29T20:30:26.737008683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:30:26.741359186Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-29T20:30:26.748408428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:30:26.751728245Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-29T20:30:27.516129378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:30:27.519518105Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:30:27.519541523Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:30:27.519564034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:30:27.522241332Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-04-29T20:30:28.10998067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:30:28.12371907Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-04-29T20:30:28.123739349Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=2
level=info ts=2026-04-29T20:30:30.289850162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:30:30.292537595Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:30:30.330585001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:30:30.337591461Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:30:31.486000302Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-04-29T20:30:32.491120318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:30:32.495102663Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:30:32.495121999Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:30:32.49514316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:30:32.497322406Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-04-29T20:30:33.11076467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:30:33.723791154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:30:33.72753105Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:30:34.86272444Z caller=checkpoint.go:372 msg="writing series" size="2.8 MB"
level=debug ts=2026-04-29T20:30:34.86274758Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=2
level=warn ts=2026-04-29T20:30:38.110155031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:30:38.235500265Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-04-29T20:30:38.472844485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:30:38.480752107Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:30:40.494480877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:30:40.497055743Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-29T20:30:41.078621597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:30:41.081974355Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:30:41.610389777Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=info ts=2026-04-29T20:30:42.765728003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:30:42.768734072Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-29T20:30:42.785902696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:30:42.792600415Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-29T20:30:43.109852568Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T20:30:43.109861619Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:30:43.109883055Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:30:43.109892162Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T20:30:44.986376578Z caller=checkpoint.go:372 msg="writing series" size="517 kB"
level=info ts=2026-04-29T20:30:45.749258115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:30:45.752185935Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
ts=2026-04-29T20:30:46.315895303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=warn ts=2026-04-29T20:30:48.110455051Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:30:48.360877763Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
level=info ts=2026-04-29T20:30:48.780060554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:30:48.784483858Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-29T20:30:51.234584209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:30:51.244735941Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:30:51.244780181Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:30:51.244818865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:30:51.25102447Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T20:30:51.735703059Z caller=checkpoint.go:372 msg="writing series" size="343 kB"
level=debug ts=2026-04-29T20:30:51.735727381Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-04-29T20:30:53.042419537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:30:53.045182001Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-04-29T20:30:53.110258594Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:30:53.270748929Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:30:55.110720422Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
level=debug ts=2026-04-29T20:30:56.037922606Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:30:56.039574147Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-29T20:30:57.576044254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:30:57.578782276Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-29T20:30:58.110258468Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:30:58.496739865Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-04-29T20:30:58.496762163Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=2
ts=2026-04-29T20:31:00.182349176Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:57954"
level=info ts=2026-04-29T20:31:00.800295931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:31:00.803882358Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-29T20:31:01.860867278Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=info ts=2026-04-29T20:31:02.052427788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:31:02.057162013Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-04-29T20:31:03.109862106Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:31:03.795962165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:31:03.802960009Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T20:31:05.23616057Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=info ts=2026-04-29T20:31:07.577954284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:31:07.582109087Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-29T20:31:08.110229455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:31:08.610981468Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=debug ts=2026-04-29T20:31:11.985914157Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-29T20:31:13.087973243Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:31:13.088477562Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:31:13.109785533Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:31:13.109804453Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:31:13.109812084Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:31:13.109866446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:31:14.072666435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:31:14.076323842Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-29T20:31:14.698240487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:31:14.705577552Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-29T20:31:15.360799591Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
ts=2026-04-29T20:31:16.318983405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=info ts=2026-04-29T20:31:16.839735171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:31:16.843205815Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-29T20:31:16.846337818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:31:16.848299728Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-29T20:31:18.110034861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:31:18.327001721Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:31:18.73619609Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-04-29T20:31:20.048192205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:31:20.053567001Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T20:31:20.418900545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:31:20.427489104Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T20:31:21.379335765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:31:21.382169256Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-29T20:31:22.112245704Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-04-29T20:31:22.112271871Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=7
level=info ts=2026-04-29T20:31:22.286481348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:31:22.289459889Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:31:22.39071325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:31:22.398130832Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T20:31:23.110392718Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:31:25.485979546Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=warn ts=2026-04-29T20:31:28.11074158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:31:28.861075645Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
level=info ts=2026-04-29T20:31:30.819207331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:31:30.82249688Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-29T20:31:32.235678649Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-04-29T20:31:33.110392393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:31:34.724827679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:31:34.734298137Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T20:31:34.865048991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:31:34.867593038Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-29T20:31:35.610782758Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=warn ts=2026-04-29T20:31:38.110660628Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:31:38.985397259Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-04-29T20:31:39.036546066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:31:39.039677067Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-29T20:31:42.065225702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:31:42.068069155Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-29T20:31:42.363989583Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-04-29T20:31:42.364014679Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=6
level=info ts=2026-04-29T20:31:42.388423634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:31:42.39596601Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-29T20:31:42.714900095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:31:42.718179925Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-29T20:31:43.110721158Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T20:31:43.110726076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:31:43.110747541Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:31:43.110755974Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-29T20:31:43.992522952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:41242"
level=info ts=2026-04-29T20:31:44.87802253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:31:44.881740122Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-29T20:31:45.383469501Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:31:45.735477407Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
ts=2026-04-29T20:31:46.322521405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=info ts=2026-04-29T20:31:46.788343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:31:46.792063865Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-29T20:31:48.110115305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:31:48.31266968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:31:48.317102041Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:31:49.112524797Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-29T20:31:49.112546656Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=debug ts=2026-04-29T20:31:52.487014526Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-29T20:31:52.48704115Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=warn ts=2026-04-29T20:31:53.110593744Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:31:53.647385957Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T20:31:55.860511537Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-04-29T20:31:56.614920736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:31:56.621507939Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-29T20:31:58.110007461Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:31:59.236340803Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=info ts=2026-04-29T20:32:00.149708438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:32:00.153418781Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:32:02.610872907Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-29T20:32:02.679150772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:32:02.682183958Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-29T20:32:03.110692671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:32:04.622284154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:32:04.625153183Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-29T20:32:05.545696965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:32:05.549811908Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-29T20:32:05.986537181Z caller=checkpoint.go:372 msg="writing series" size="845 kB"
level=debug ts=2026-04-29T20:32:05.986559259Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=warn ts=2026-04-29T20:32:08.110252292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:32:08.567860894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:32:08.574371032Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T20:32:09.360690268Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-04-29T20:32:10.745562216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:32:10.752080167Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T20:32:11.314927071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:32:11.317843643Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-29T20:32:12.735504728Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-04-29T20:32:13.088131691Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:32:13.088559777Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:32:13.109828724Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:32:13.109853179Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:32:13.109861717Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:32:13.109936048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:32:14.755408237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:32:14.762312229Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T20:32:15.363657057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:32:15.366743209Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:32:15.366764299Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:32:15.366783659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:32:15.369209612Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-29T20:32:16.110821256Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
ts=2026-04-29T20:32:16.325360511Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=info ts=2026-04-29T20:32:16.390014547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="944 B" avg_comp="944 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:32:16.392601933Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:32:16.392624412Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:32:16.392645627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:32:16.394555104Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:32:16.582169845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:32:16.584703365Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-29T20:32:18.015868464Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:32:18.01591567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="446 B" avg_comp="446 B" total_uncomp="36 kB" avg_uncomp="36 kB" idle=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:32:18.018697466Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-04-29T20:32:18.110030613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:32:19.486266784Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-04-29T20:32:22.861131641Z caller=checkpoint.go:372 msg="writing series" size="347 kB"
level=warn ts=2026-04-29T20:32:23.110762511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:32:26.23619093Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-04-29T20:32:26.424731871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:32:26.428259888Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-29T20:32:26.703173731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:32:26.705974663Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-04-29T20:32:26.705993585Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:32:26.706013752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:32:26.70816491Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-29T20:32:28.110396865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:32:29.218126569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:32:29.221285473Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:32:29.316628234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:32:29.31948183Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:32:29.319503751Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:32:29.319528126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:32:29.321689804Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-29T20:32:29.611296581Z caller=checkpoint.go:372 msg="writing series" size="627 kB"
level=debug ts=2026-04-29T20:32:29.611315608Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=7
level=info ts=2026-04-29T20:32:31.302923555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:32:31.306488356Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:32:32.986840519Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=warn ts=2026-04-29T20:32:33.110748764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:32:33.360490897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:32:33.363335778Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-29T20:32:36.361141008Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=debug ts=2026-04-29T20:32:36.36117436Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
ts=2026-04-29T20:32:36.963671156Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:45736"
level=info ts=2026-04-29T20:32:37.002189933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:32:37.006900584Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-29T20:32:37.794310161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:32:37.797456327Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-04-29T20:32:38.110075238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:32:39.265628983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:32:39.272804514Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:32:39.793208683Z caller=checkpoint.go:372 msg="writing series" size="59 MB"
level=debug ts=2026-04-29T20:32:39.793252726Z caller=checkpoint.go:384 msg="flushing series" totalSize="59 MB" series=1
level=info ts=2026-04-29T20:32:41.164662705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:32:41.168774502Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T20:32:43.110169559Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T20:32:43.11018359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:32:43.110203968Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:32:43.110213362Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:32:43.115682258Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000013.tmp new=/var/loki/wal/checkpoint.000013
level=info ts=2026-04-29T20:32:43.117083531Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006857645s
level=debug ts=2026-04-29T20:32:43.826755353Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:32:43.837748048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:32:43.842074215Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-04-29T20:32:43.998497812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:50632"
level=info ts=2026-04-29T20:32:45.101154558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:32:45.104961834Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-29T20:32:46.328053273Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=warn ts=2026-04-29T20:32:48.110242314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:32:48.308748399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:32:48.315710832Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-29T20:32:48.853839647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:32:48.862710524Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:32:48.862734613Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:32:48.86275981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:32:48.868860244Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:32:49.057404047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:32:49.060167228Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-04-29T20:32:53.109883798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:32:53.639281679Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-29T20:32:55.340444547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:32:55.34304667Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:32:55.343066947Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:32:55.343087299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:32:55.345029092Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-29T20:32:56.415222316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:32:56.419365622Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-29T20:32:58.030092381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:32:58.034157684Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-29T20:32:58.110490545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:32:58.26889514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:32:58.275988259Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T20:32:59.338695391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:32:59.341621063Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-29T20:32:59.345689547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:32:59.352886549Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T20:33:00.128627749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:33:00.136114502Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T20:33:03.109916162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:33:03.343496531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:33:03.346725656Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-29T20:33:04.608962658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:33:04.611617889Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-29T20:33:06.444256531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:33:06.446922997Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:33:06.446944628Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:33:06.446967268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:33:06.450014702Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-29T20:33:08.11039402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:33:10.483843131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:33:10.487858195Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:33:10.487882395Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:33:10.487903752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:33:10.490281784Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-29T20:33:10.843110672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:33:10.845919632Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:33:10.845939692Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:33:10.845960903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:33:10.84810584Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-29T20:33:10.876329843Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:33:12.170188174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:33:12.17280797Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-29T20:33:13.089470458Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:33:13.093652976Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=warn ts=2026-04-29T20:33:13.109850847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:33:13.109864609Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-29T20:33:13.109877161Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-04-29T20:33:13.109895232Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:33:13.109907523Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:33:13.110098143Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000014
level=info ts=2026-04-29T20:33:13.110107223Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-29T20:33:13.110127708Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.342µs success=true
level=debug ts=2026-04-29T20:33:13.11035144Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-04-29T20:33:13.131118726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:33:13.133521286Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-04-29T20:33:16.331484884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T20:33:16.485888537Z caller=checkpoint.go:372 msg="writing series" size="294 kB"
level=info ts=2026-04-29T20:33:17.885769872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:33:17.888693877Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-29T20:33:17.966777022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:33:17.969467124Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-29T20:33:18.110324268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:33:19.861365805Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-04-29T20:33:23.021877189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:33:23.027123026Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-04-29T20:33:23.110388505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:33:23.235796412Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-04-29T20:33:24.179098908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:33:24.183167491Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T20:33:25.174469702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:33:25.178573629Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:33:26.610703661Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=info ts=2026-04-29T20:33:27.275395717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:33:27.278673898Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-04-29T20:33:28.110380005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:33:28.539706012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:33:28.543317628Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-29T20:33:28.920425633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:33:28.927674043Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-29T20:33:29.985872534Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-04-29T20:33:31.504901147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:33:31.513566847Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:33:31.513588537Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:33:31.513609783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:33:31.519654013Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-29T20:33:32.027014223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:33:32.030826386Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-29T20:33:33.109873924Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:33:33.360622402Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=info ts=2026-04-29T20:33:35.654828645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:33:35.661727015Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T20:33:36.736052696Z caller=checkpoint.go:372 msg="writing series" size="5.1 kB"
level=info ts=2026-04-29T20:33:37.081030317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:33:37.084068002Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:33:37.084088068Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:33:37.084109468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:33:37.086625569Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-29T20:33:37.355567205Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:33:37.384698974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:33:37.387777513Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-29T20:33:37.548961749Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-29T20:33:38.110550651Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:33:40.1110878Z caller=checkpoint.go:372 msg="writing series" size="641 kB"
level=debug ts=2026-04-29T20:33:40.111111982Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=9
level=info ts=2026-04-29T20:33:40.329330355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:33:40.332731304Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-29T20:33:40.811347134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:33:40.819521613Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T20:33:42.439068722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:33:42.445899901Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T20:33:42.561271851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:33:42.564030358Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:33:43.110726743Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:33:43.110765903Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-29T20:33:43.110753311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:33:43.11077553Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:33:43.221239462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:33:43.228146663Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:33:43.488782439Z caller=checkpoint.go:372 msg="writing series" size="4.0 MB"
level=debug ts=2026-04-29T20:33:43.488803436Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.0 MB" series=1
ts=2026-04-29T20:33:46.334236907Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T20:33:46.860885245Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-04-29T20:33:48.110745201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:33:48.615875916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="889 B" avg_comp="889 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:33:48.6189726Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=debug ts=2026-04-29T20:33:50.236054159Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=info ts=2026-04-29T20:33:51.635557926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:33:51.638388251Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-29T20:33:52.070586378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:33:52.074092438Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-29T20:33:53.110459215Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:33:53.611070726Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=info ts=2026-04-29T20:33:54.64742223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:33:54.651656198Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-29T20:33:55.31294355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:33:55.315757732Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-29T20:33:56.076722977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:33:56.080072998Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-29T20:33:56.096384712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:33:56.098770156Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-29T20:33:56.986348875Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-29T20:33:58.110629791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T20:34:00.197495085Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:58256"
level=debug ts=2026-04-29T20:34:00.360614382Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-04-29T20:34:00.582318058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:34:00.586745304Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-29T20:34:02.286189809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:34:02.289158343Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-29T20:34:03.110242135Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:34:03.735744161Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-04-29T20:34:03.736787751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:34:03.740359715Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-29T20:34:06.578642858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:34:06.582641523Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-29T20:34:06.659735539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:34:06.667113913Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-29T20:34:06.970912411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:32914"
level=debug ts=2026-04-29T20:34:07.111391612Z caller=checkpoint.go:372 msg="writing series" size="171 kB"
level=warn ts=2026-04-29T20:34:08.110470386Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:34:08.861649028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:34:08.865627815Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:34:08.865648858Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:34:08.865669921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:34:08.86942658Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T20:34:09.865151191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:34:09.872660314Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-29T20:34:10.120117844Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:34:10.486268341Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=info ts=2026-04-29T20:34:13.087975979Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:34:13.087997325Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:34:13.11040539Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:34:13.110425159Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:34:13.110450291Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:34:13.110420821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:34:13.860875782Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
ts=2026-04-29T20:34:14.006767545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:55188"
ts=2026-04-29T20:34:16.337715147Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T20:34:17.235741435Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=info ts=2026-04-29T20:34:17.256866718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:34:17.259774456Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-29T20:34:17.419822149Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-29T20:34:18.109935977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:34:19.79812592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:34:19.801642898Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-29T20:34:20.314634829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:34:20.318266203Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:34:20.61150023Z caller=checkpoint.go:372 msg="writing series" size="532 kB"
level=debug ts=2026-04-29T20:34:20.611529269Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=11
level=info ts=2026-04-29T20:34:22.22473057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:34:22.23231044Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:34:22.945411833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:34:22.949001185Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-04-29T20:34:23.110514604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:34:23.437125847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:34:23.439812734Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-29T20:34:23.986097195Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=info ts=2026-04-29T20:34:26.510208012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:34:26.516227889Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-29T20:34:27.360709755Z caller=checkpoint.go:372 msg="writing series" size="6.5 kB"
level=warn ts=2026-04-29T20:34:28.110263379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:34:30.541712674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:34:30.548355576Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:34:30.735954562Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-29T20:34:33.110238246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:34:34.110599334Z caller=checkpoint.go:372 msg="writing series" size="500 kB"
level=info ts=2026-04-29T20:34:36.331979293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:34:36.338956719Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T20:34:36.612653655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:34:36.615800859Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-04-29T20:34:36.97409276Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:59590"
level=debug ts=2026-04-29T20:34:37.485766161Z caller=checkpoint.go:372 msg="writing series" size="598 kB"
level=debug ts=2026-04-29T20:34:37.485788882Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-04-29T20:34:38.110066274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:34:39.498194322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:34:39.501333025Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-29T20:34:39.658324233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:34:39.66099892Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-29T20:34:40.861086052Z caller=checkpoint.go:372 msg="writing series" size="245 kB"
level=debug ts=2026-04-29T20:34:40.931503877Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:34:43.110538268Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:34:43.110565941Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:34:43.110573709Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:34:43.110626316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:34:43.307214447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:34:43.310593193Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-04-29T20:34:44.00947575Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:47030"
level=debug ts=2026-04-29T20:34:44.236494743Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=info ts=2026-04-29T20:34:44.442926111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:34:44.445966327Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-04-29T20:34:46.340196241Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=info ts=2026-04-29T20:34:46.595285027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:34:46.59926657Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-29T20:34:46.902181102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:34:46.904721325Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=debug ts=2026-04-29T20:34:47.61170217Z caller=checkpoint.go:372 msg="writing series" size="761 kB"
level=debug ts=2026-04-29T20:34:47.611723726Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=warn ts=2026-04-29T20:34:48.110655986Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:34:48.621489127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:34:48.625899343Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T20:34:48.898246533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:34:48.901156989Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-29T20:34:50.892689672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:34:50.896362705Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:34:50.896378996Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:34:50.896398844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:34:50.899500677Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:34:50.985911636Z caller=checkpoint.go:372 msg="writing series" size="280 kB"
level=warn ts=2026-04-29T20:34:53.110527338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:34:54.202740736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:34:54.207787109Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:34:54.361190481Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-04-29T20:34:57.736244486Z caller=checkpoint.go:372 msg="writing series" size="745 kB"
level=warn ts=2026-04-29T20:34:58.109888487Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:34:59.866600869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:34:59.870657279Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-29T20:35:01.110542311Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=debug ts=2026-04-29T20:35:01.110566656Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-04-29T20:35:03.070078597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:35:03.07693151Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-29T20:35:03.109830404Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:35:03.235216395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:35:03.238040256Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-29T20:35:04.485856793Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=info ts=2026-04-29T20:35:06.82147016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:35:06.829413293Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:35:06.829446992Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:35:06.829469452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:35:06.835097282Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-04-29T20:35:06.977323007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:58714"
level=debug ts=2026-04-29T20:35:07.211500857Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:35:07.861266191Z caller=checkpoint.go:372 msg="writing series" size="653 B"
level=warn ts=2026-04-29T20:35:08.11032741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:35:09.539198738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:35:09.543079629Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:35:11.236015151Z caller=checkpoint.go:372 msg="writing series" size="591 kB"
level=info ts=2026-04-29T20:35:13.087987498Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:35:13.088005154Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:35:13.110411724Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:35:13.1104448Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-29T20:35:13.110422755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:35:13.110452915Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T20:35:13.555652893Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:35:13.557345238Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-29T20:35:14.610653724Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-04-29T20:35:14.765017431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:35:14.767747027Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-04-29T20:35:16.342655604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=info ts=2026-04-29T20:35:16.952750674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:35:16.956035394Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-29T20:35:17.733720365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:35:17.740304742Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T20:35:17.803476777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:35:17.806918029Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-29T20:35:17.985584215Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=warn ts=2026-04-29T20:35:18.109928079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:35:19.253657925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:35:19.26014758Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:35:21.16697842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:35:21.173200262Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-29T20:35:21.360941154Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=debug ts=2026-04-29T20:35:21.360966555Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-04-29T20:35:21.979838102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:35:21.982757154Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-29T20:35:23.110469226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:35:24.472556975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:35:24.476241485Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-29T20:35:24.735786028Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-04-29T20:35:25.579829914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:35:25.582796852Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-29T20:35:28.10990725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:35:28.121912672Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-04-29T20:35:28.121931499Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=2
level=info ts=2026-04-29T20:35:29.196307399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:35:29.204193009Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T20:35:31.48621475Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-04-29T20:35:33.11054869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:35:33.611706235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:35:33.615307495Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-29T20:35:33.691769468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:35:33.694458264Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-29T20:35:34.190806076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:35:34.193468882Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-29T20:35:34.863410761Z caller=checkpoint.go:372 msg="writing series" size="2.8 MB"
level=debug ts=2026-04-29T20:35:34.863430504Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=2
level=info ts=2026-04-29T20:35:34.902768913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:35:34.905550218Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-29T20:35:36.761639708Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T20:35:38.110683945Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:35:38.236016484Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-04-29T20:35:41.55364308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:35:41.558357504Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-29T20:35:41.610607908Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=info ts=2026-04-29T20:35:41.667705254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:35:41.671333636Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-29T20:35:42.595184522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:35:42.597909925Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-29T20:35:43.110749749Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T20:35:43.110749689Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:35:43.110784083Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:35:43.110795269Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:35:43.481336894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:35:43.48467515Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-29T20:35:44.361465924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:35:44.36440317Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-29T20:35:44.986901603Z caller=checkpoint.go:372 msg="writing series" size="477 kB"
ts=2026-04-29T20:35:46.344936478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=info ts=2026-04-29T20:35:48.021819362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:35:48.025585112Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-29T20:35:48.110768327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:35:48.360457594Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=info ts=2026-04-29T20:35:50.180549643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:35:50.188722131Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:35:51.736357595Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-04-29T20:35:53.110768125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:35:55.111039107Z caller=checkpoint.go:372 msg="writing series" size="317 kB"
level=debug ts=2026-04-29T20:35:55.111061855Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-04-29T20:35:55.74758082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:35:55.757047475Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-29T20:35:58.110205164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:35:58.467573184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:35:58.476798985Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:35:58.476825677Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:35:58.476861643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:35:58.48306405Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:35:58.497816639Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-04-29T20:35:58.497831051Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=1
level=info ts=2026-04-29T20:35:59.772177805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:35:59.779598095Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:36:00.529316265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:36:00.532730304Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-29T20:36:00.938992304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:36:00.94251716Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-29T20:36:01.860476078Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=warn ts=2026-04-29T20:36:03.109971778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:36:03.239079689Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T20:36:05.235671975Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=info ts=2026-04-29T20:36:07.126267729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:36:07.129747284Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-29T20:36:07.155945867Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:36:07.835151313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:36:07.838008512Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-29T20:36:08.110701436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:36:08.610769767Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-04-29T20:36:11.854818836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:36:11.858826649Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-29T20:36:11.986520854Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-04-29T20:36:13.087811437Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:36:13.088856949Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-29T20:36:13.110222055Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:36:13.110257988Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:36:13.110268922Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:36:13.117586875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T20:36:14.015162772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:42636"
level=debug ts=2026-04-29T20:36:15.360545042Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=info ts=2026-04-29T20:36:16.129773911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:36:16.136669408Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
ts=2026-04-29T20:36:16.347319289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=info ts=2026-04-29T20:36:16.683640996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:36:16.68652621Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-29T20:36:17.777501824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:36:17.784337917Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T20:36:18.110020936Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:36:18.735888101Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-04-29T20:36:21.288330551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:36:21.291318123Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-29T20:36:22.111317356Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-04-29T20:36:22.11134091Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=7
level=info ts=2026-04-29T20:36:22.550888002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:36:22.55417862Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-04-29T20:36:23.109982597Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:36:25.485920907Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=info ts=2026-04-29T20:36:25.758460653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:36:25.761828995Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-29T20:36:28.110472787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:36:28.861185266Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
ts=2026-04-29T20:36:30.210170892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:58366"
level=info ts=2026-04-29T20:36:30.617480997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:36:30.62100719Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-29T20:36:31.845494122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:36:31.849774038Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-29T20:36:32.236046474Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-04-29T20:36:32.856741444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:36:32.863418109Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-29T20:36:33.110486366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:36:33.985703144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:36:33.988290226Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-29T20:36:34.294330953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:36:34.297499295Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-29T20:36:35.611224764Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=info ts=2026-04-29T20:36:35.783830069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:36:35.787957657Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-29T20:36:37.533719944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:36:37.536652551Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:36:37.536672721Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:36:37.536703513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:36:37.53911447Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-04-29T20:36:38.110414694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:36:38.986126862Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=debug ts=2026-04-29T20:36:39.489961248Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T20:36:40.428255187Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:36:42.064781628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:36:42.069110949Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:36:42.069131236Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:36:42.06915211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:36:42.072278623Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-29T20:36:42.362620245Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-04-29T20:36:42.362643615Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=6
level=info ts=2026-04-29T20:36:43.109802445Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:36:43.1098339Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:36:43.109841621Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:36:43.109846524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:36:45.735904511Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
ts=2026-04-29T20:36:46.350613903Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=info ts=2026-04-29T20:36:46.695028386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:36:46.698913746Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-29T20:36:48.110545549Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:36:49.114332134Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-04-29T20:36:49.114359672Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=2
level=info ts=2026-04-29T20:36:51.388344148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:36:51.391250906Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-29T20:36:52.100038111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:36:52.107503688Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:36:52.486791008Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-29T20:36:52.486815068Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=warn ts=2026-04-29T20:36:53.110751806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:36:54.122124163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:36:54.129979109Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:36:54.669560312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:36:54.673000294Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-29T20:36:55.567248187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:36:55.574913829Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-29T20:36:55.86071557Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-04-29T20:36:58.110399496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:36:58.392199682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:36:58.397567953Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-29T20:36:59.236478305Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=info ts=2026-04-29T20:36:59.855012705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:36:59.858390312Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-04-29T20:37:00.213018075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:53620"
level=info ts=2026-04-29T20:37:00.980718567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:37:00.983416888Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-29T20:37:02.611552468Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-04-29T20:37:03.110828967Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:37:05.986086902Z caller=checkpoint.go:372 msg="writing series" size="889 kB"
level=debug ts=2026-04-29T20:37:05.986108224Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-04-29T20:37:07.575565468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:37:07.583981705Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:37:07.584006884Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:37:07.584033639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:37:07.590171349Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=warn ts=2026-04-29T20:37:08.110717472Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:37:08.509242991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-29T20:37:08.510015047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:37:08.512131306Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:37:08.512130853Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-29T20:37:09.361233102Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-04-29T20:37:10.667836193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="944 B" avg_comp="944 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:37:10.671202816Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-29T20:37:12.229096579Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:37:12.735691328Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-04-29T20:37:13.088123962Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:37:13.088567007Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=warn ts=2026-04-29T20:37:13.109870836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:37:13.109882384Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:37:13.109901006Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:37:13.109910314Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:37:15.732775986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:37:15.738085307Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-29T20:37:16.110609456Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
ts=2026-04-29T20:37:16.353001788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=warn ts=2026-04-29T20:37:18.110545169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:37:19.485593Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-04-29T20:37:19.841482875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:37:19.844416731Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:37:19.844452902Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:37:19.844473749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:37:19.846703602Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-29T20:37:20.829719312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:37:20.832672818Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-29T20:37:21.362241549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:37:21.365023301Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-29T20:37:22.567666027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:37:22.570207763Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-29T20:37:22.86127068Z caller=checkpoint.go:372 msg="writing series" size="347 kB"
level=warn ts=2026-04-29T20:37:23.110357399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:37:23.518280829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:37:23.52477937Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T20:37:25.030568839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:37:25.033724113Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:37:26.235352039Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-04-29T20:37:26.260216373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:37:26.263245713Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-29T20:37:27.225729021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:37:27.22886439Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-29T20:37:28.110216379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:37:28.266421594Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T20:37:29.611483823Z caller=checkpoint.go:372 msg="writing series" size="627 kB"
level=debug ts=2026-04-29T20:37:29.611503249Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=7
ts=2026-04-29T20:37:30.216691027Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:58136"
level=info ts=2026-04-29T20:37:31.761677411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:37:31.772479481Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:37:32.986846981Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-04-29T20:37:32.986869511Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=1
level=warn ts=2026-04-29T20:37:33.110784807Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:37:34.965846725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:37:34.968612262Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-29T20:37:35.550467274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:37:35.554003721Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-29T20:37:36.103684452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:37:36.110876591Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T20:37:36.360692506Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=info ts=2026-04-29T20:37:37.272186606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:37:37.275343079Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-04-29T20:37:38.110864412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:37:38.600988506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:37:38.605008144Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:37:39.793086546Z caller=checkpoint.go:372 msg="writing series" size="59 MB"
level=debug ts=2026-04-29T20:37:39.793120289Z caller=checkpoint.go:384 msg="flushing series" totalSize="60 MB" series=2
level=info ts=2026-04-29T20:37:40.009765978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:37:40.013267056Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-29T20:37:43.110472088Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:37:43.110509519Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:37:43.110517139Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:37:43.110627508Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:37:43.112409834Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000014.tmp new=/var/loki/wal/checkpoint.000014
level=info ts=2026-04-29T20:37:43.113831553Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.003559997s
level=info ts=2026-04-29T20:37:43.377067716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:37:43.384018039Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-29T20:37:44.021290667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:37618"
level=info ts=2026-04-29T20:37:44.66668838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:37:44.673634508Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-29T20:37:45.095812732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:37:45.098557451Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-04-29T20:37:46.356221433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T20:37:46.803187243Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T20:37:48.110147162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:37:52.894398108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:37:52.902774892Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=warn ts=2026-04-29T20:37:53.110490745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:37:53.191801044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:37:53.194736823Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-04-29T20:37:58.10988858Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:38:00.116826403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:38:00.120335366Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-29T20:38:02.35423863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:38:02.360234193Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-29T20:38:03.10991051Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T20:38:06.989372328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:41706"
level=warn ts=2026-04-29T20:38:08.110100623Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:38:08.401316495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:38:08.404721801Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:38:10.443065129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:38:10.447388527Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-29T20:38:11.848487546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:38:11.851458901Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-29T20:38:13.08778258Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:38:13.088830388Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-29T20:38:13.110235017Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-04-29T20:38:13.110231197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:38:13.110227456Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:38:13.110257566Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:38:13.110269935Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:38:13.110372379Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000015
level=info ts=2026-04-29T20:38:13.110381125Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-29T20:38:13.110398079Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.955µs success=true
level=debug ts=2026-04-29T20:38:13.110616369Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
ts=2026-04-29T20:38:14.023818483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:50370"
level=info ts=2026-04-29T20:38:14.74253575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:38:14.745152565Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-29T20:38:15.786486971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:38:15.789294687Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-29T20:38:16.203660381Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:38:16.229696303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:38:16.233489313Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-04-29T20:38:16.358878659Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:38:16.47688983Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T20:38:16.48637774Z caller=checkpoint.go:372 msg="writing series" size="294 kB"
level=info ts=2026-04-29T20:38:18.015642346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:38:18.018591961Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-29T20:38:18.110106167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:38:19.839015465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:38:19.841742602Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-29T20:38:19.860843985Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-04-29T20:38:21.9958417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:38:21.999120216Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-29T20:38:23.100107688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:38:23.10451622Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-04-29T20:38:23.110695706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:38:23.236069705Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-04-29T20:38:26.069773074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:38:26.073342597Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:38:26.611328303Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=warn ts=2026-04-29T20:38:28.110816943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:38:28.641940274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:38:28.649614253Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-29T20:38:29.594542102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:38:29.600853444Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T20:38:29.986547858Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-04-29T20:38:33.049816206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:38:33.053551909Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-29T20:38:33.110516634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:38:33.361324789Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=info ts=2026-04-29T20:38:36.056026002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:38:36.059097731Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:38:36.059119048Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:38:36.059138601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:38:36.061499632Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-29T20:38:36.736520816Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
ts=2026-04-29T20:38:36.992474297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:49298"
level=warn ts=2026-04-29T20:38:38.110214493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:38:39.925482457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:38:39.932520395Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-29T20:38:40.111773108Z caller=checkpoint.go:372 msg="writing series" size="641 kB"
level=debug ts=2026-04-29T20:38:40.111798388Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=9
level=info ts=2026-04-29T20:38:43.109806755Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:38:43.109841022Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:38:43.109849352Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:38:43.109875809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:38:43.209233016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:38:43.21741521Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T20:38:43.485717144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:38:43.488905436Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-29T20:38:43.489346753Z caller=checkpoint.go:372 msg="writing series" size="4.0 MB"
level=debug ts=2026-04-29T20:38:43.489361916Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.0 MB" series=1
level=info ts=2026-04-29T20:38:44.677737687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:38:44.680502136Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-04-29T20:38:46.36170503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T20:38:46.861403979Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-04-29T20:38:47.334985667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:38:47.342735936Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=warn ts=2026-04-29T20:38:48.110848181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:38:49.105930732Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:38:49.813458584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:38:49.816230408Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-29T20:38:50.236600371Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
level=info ts=2026-04-29T20:38:52.339946464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="889 B" avg_comp="889 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:38:52.342572346Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-04-29T20:38:53.110160261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:38:53.612008162Z caller=checkpoint.go:372 msg="writing series" size="532 kB"
level=info ts=2026-04-29T20:38:53.733948255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:38:53.737488909Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-29T20:38:54.045599958Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:38:54.047347286Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-29T20:38:55.173653161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:38:55.17646547Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-29T20:38:56.986651565Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-29T20:38:58.110857834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:39:00.006903043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:39:00.010331338Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:39:00.01035198Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:39:00.010372939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:39:00.01260562Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-29T20:39:00.360739635Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=info ts=2026-04-29T20:39:00.38995042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:39:00.392836243Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-04-29T20:39:00.392858733Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:39:00.392883901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:39:00.395170066Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-29T20:39:03.109857666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:39:03.736376194Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-04-29T20:39:06.734128567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:39:06.737995887Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:39:07.111570326Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=info ts=2026-04-29T20:39:07.920660175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:39:07.924083046Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-04-29T20:39:08.110543048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:39:09.413550549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:39:09.417283623Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:39:10.486509458Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=info ts=2026-04-29T20:39:10.587822384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:39:10.595903167Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:39:10.595928058Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:39:10.595957704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:39:10.602867164Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:39:10.96678087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:39:10.969379548Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:39:10.969400373Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:39:10.969420167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:39:10.971518343Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-29T20:39:11.562421964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:39:11.56499158Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:39:11.565020686Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:39:11.565042183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:39:11.567077885Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-29T20:39:12.900981719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:39:12.903867471Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:39:13.087898196Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:39:13.088278436Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:39:13.110683486Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:39:13.110706497Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:39:13.110714705Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:39:13.110711733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:39:13.438943665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:39:13.442254414Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-29T20:39:13.861596379Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=debug ts=2026-04-29T20:39:13.861621197Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-04-29T20:39:13.864618508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:39:13.871677592Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-29T20:39:14.9513393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:39:14.954232512Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:39:14.954258097Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:39:14.954282981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:39:14.956915669Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-04-29T20:39:16.364190532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T20:39:17.236403541Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=debug ts=2026-04-29T20:39:18.10551414Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T20:39:18.110525548Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:39:19.630096918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:39:19.632997796Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-29T20:39:20.61177168Z caller=checkpoint.go:372 msg="writing series" size="532 kB"
level=info ts=2026-04-29T20:39:23.028957816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:39:23.032509973Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-04-29T20:39:23.109886314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:39:23.471680442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:39:23.478362369Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T20:39:23.986745667Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=info ts=2026-04-29T20:39:26.976050374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:39:26.979170837Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-29T20:39:27.360866512Z caller=checkpoint.go:372 msg="writing series" size="6.5 kB"
level=warn ts=2026-04-29T20:39:28.110282798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:39:28.862696804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:39:28.865627159Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-29T20:39:29.704953784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:39:29.708584841Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-29T20:39:30.736512282Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-29T20:39:33.110880104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:39:33.547322525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:39:33.555253259Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-29T20:39:33.752864453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:39:33.755717612Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-29T20:39:34.111254859Z caller=checkpoint.go:372 msg="writing series" size="637 kB"
level=debug ts=2026-04-29T20:39:34.111279321Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=info ts=2026-04-29T20:39:34.52248943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:39:34.525343194Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-29T20:39:36.903072316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:39:36.909450332Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:39:37.486716173Z caller=checkpoint.go:372 msg="writing series" size="494 kB"
level=warn ts=2026-04-29T20:39:38.110065136Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:39:40.381335725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:39:40.384547966Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:39:40.384569649Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:39:40.38459367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:39:40.387044328Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-29T20:39:40.861366992Z caller=checkpoint.go:372 msg="writing series" size="245 kB"
level=info ts=2026-04-29T20:39:41.50541947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:39:41.508344608Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:39:41.508362644Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:39:41.508384572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:39:41.510580447Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-29T20:39:42.693182031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:39:42.696030901Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-29T20:39:43.109782639Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:39:43.109820735Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:39:43.109830884Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:39:43.109856666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:39:44.235888217Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=info ts=2026-04-29T20:39:44.248974624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:39:44.256220021Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
ts=2026-04-29T20:39:46.36669726Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:39:47.61170763Z caller=checkpoint.go:372 msg="writing series" size="761 kB"
level=debug ts=2026-04-29T20:39:47.611731496Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=4
level=info ts=2026-04-29T20:39:47.746743422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:39:47.750634923Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:39:47.943313999Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T20:39:48.110406356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:39:49.121785836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:39:49.124904552Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:39:49.124924515Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:39:49.124944986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:39:49.12757031Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-29T20:39:49.8921837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:39:49.894379958Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:39:49.894402744Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:39:49.89442304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:39:49.896417629Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-29T20:39:50.14832398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:39:50.150705887Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-29T20:39:50.986185117Z caller=checkpoint.go:372 msg="writing series" size="280 kB"
level=info ts=2026-04-29T20:39:51.022137387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:39:51.028174431Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:39:51.25285841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:39:51.255496608Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-29T20:39:51.787044902Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-29T20:39:53.110109741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:39:53.656553498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:39:53.659155199Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-29T20:39:54.20986697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:39:54.212308721Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-29T20:39:54.360700767Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-04-29T20:39:56.017224005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:39:56.023696654Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:39:56.023716574Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:39:56.023739056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:39:56.029898182Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-29T20:39:56.588758826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:39:56.592217331Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:39:57.736390645Z caller=checkpoint.go:372 msg="writing series" size="745 kB"
level=warn ts=2026-04-29T20:39:58.110666098Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T20:40:00.228206634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:38516"
level=debug ts=2026-04-29T20:40:01.111630765Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=debug ts=2026-04-29T20:40:01.111655089Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=warn ts=2026-04-29T20:40:03.109926792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:40:03.274321125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:40:03.278765109Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-29T20:40:03.922488155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:40:03.926558035Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-04-29T20:40:03.926582034Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:40:03.926608155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:40:03.929580426Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-29T20:40:04.486212658Z caller=checkpoint.go:372 msg="writing series" size="229 kB"
level=info ts=2026-04-29T20:40:04.822061712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:40:04.825590837Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-29T20:40:05.44341737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:40:05.446154305Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-29T20:40:06.340383482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:40:06.343214485Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-29T20:40:07.401207889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:40:07.404158948Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-29T20:40:07.861247671Z caller=checkpoint.go:372 msg="writing series" size="653 B"
level=warn ts=2026-04-29T20:40:08.110226805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:40:08.202554994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:40:08.206180215Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-29T20:40:11.236057785Z caller=checkpoint.go:372 msg="writing series" size="592 kB"
level=info ts=2026-04-29T20:40:13.088563315Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:40:13.089188646Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:40:13.110705904Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:40:13.110731309Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:40:13.11073895Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:40:13.110789362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:40:14.503960292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:40:14.512242926Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:40:14.611635599Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
ts=2026-04-29T20:40:16.369458342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T20:40:17.985739578Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=debug ts=2026-04-29T20:40:17.985764716Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-04-29T20:40:18.110366132Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:40:19.797603109Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:40:19.797591188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4b0eaddb922ae234 immediate=false num_chunks=1 total_comp="175 kB" avg_comp="175 kB" total_uncomp="885 kB" avg_uncomp="885 kB" idle=1 labels="{stream=\"stat-rep-ldla\"}"
level=error ts=2026-04-29T20:40:19.801325066Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4b0eaddb922ae234/19ddab8ac16%3A19ddac127f7%3Aacbba898?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-ldla\"}"
level=info ts=2026-04-29T20:40:20.522572185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:40:20.526412065Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-29T20:40:21.361524242Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=info ts=2026-04-29T20:40:21.983402578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:40:21.986507618Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-29T20:40:22.303489074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:40:22.307476938Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-29T20:40:22.73648325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:40:22.742700652Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-29T20:40:23.110145254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:40:23.788991563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:40:23.791839425Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-29T20:40:24.31448418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:40:24.317361823Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=debug ts=2026-04-29T20:40:24.736657543Z caller=checkpoint.go:372 msg="writing series" size="8.7 kB"
level=debug ts=2026-04-29T20:40:26.88106635Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-29T20:40:26.920172267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:40:26.922608841Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-04-29T20:40:28.110174217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:40:28.125055066Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-04-29T20:40:28.125074555Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=3
level=info ts=2026-04-29T20:40:29.82959017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:40:29.836840307Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
ts=2026-04-29T20:40:30.231602732Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:51102"
level=debug ts=2026-04-29T20:40:31.486179285Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-04-29T20:40:32.050563636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:40:32.054029979Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:40:32.054054377Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:40:32.054079552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:40:32.057156255Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-29T20:40:33.11048251Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:40:33.996359207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4b0eaddb922ae234 immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="885 kB" avg_uncomp="885 kB" full=1 labels="{stream=\"stat-rep-ldla\"}"
level=error ts=2026-04-29T20:40:34.000874405Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4b0eaddb922ae234/19ddab8ac16%3A19ddac127f7%3Aacbba898?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-ldla\"}"
level=info ts=2026-04-29T20:40:34.158401061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:40:34.161059606Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-29T20:40:34.862844324Z caller=checkpoint.go:372 msg="writing series" size="2.9 MB"
level=debug ts=2026-04-29T20:40:34.862866929Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=2
level=info ts=2026-04-29T20:40:35.476051059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:40:35.48377672Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-29T20:40:36.99972763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:35324"
level=info ts=2026-04-29T20:40:37.797880171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:40:37.804542368Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T20:40:38.110383949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:40:38.235747521Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-04-29T20:40:40.03907624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:40:40.04195981Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:40:40.389084225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:40:40.392594676Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:40:41.61086676Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=info ts=2026-04-29T20:40:42.03312855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:40:42.036850939Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-29T20:40:43.110000712Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:40:43.110031072Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:40:43.110039668Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:40:43.11004306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:40:44.986977573Z caller=checkpoint.go:372 msg="writing series" size="558 kB"
level=info ts=2026-04-29T20:40:46.116956518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:40:46.124688261Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-29T20:40:46.372277035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=info ts=2026-04-29T20:40:47.869755719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:40:47.872706983Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-04-29T20:40:48.110516649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:40:48.36105127Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=debug ts=2026-04-29T20:40:48.866815372Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:40:50.728491917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:40:50.732722062Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:40:51.736570368Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
level=warn ts=2026-04-29T20:40:53.110033796Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:40:53.933627771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:40:53.937656141Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-29T20:40:54.335579316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:40:54.33901226Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-29T20:40:54.771473606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:40:54.774231203Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-29T20:40:55.111886578Z caller=checkpoint.go:372 msg="writing series" size="383 kB"
level=debug ts=2026-04-29T20:40:55.111909441Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=warn ts=2026-04-29T20:40:58.110302615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:40:58.499477143Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-04-29T20:40:58.499502052Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=1
level=info ts=2026-04-29T20:41:00.097398476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:41:00.100529981Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-29T20:41:01.861691079Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=warn ts=2026-04-29T20:41:03.110676573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:41:03.283499028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:41:03.286688805Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-29T20:41:05.236334023Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=info ts=2026-04-29T20:41:05.595637699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:41:05.599281898Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-04-29T20:41:07.002993861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:33530"
level=info ts=2026-04-29T20:41:07.41255598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:41:07.415634226Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-04-29T20:41:08.110787776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:41:08.11793097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:41:08.12386898Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-29T20:41:08.611155295Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=debug ts=2026-04-29T20:41:11.986125349Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-29T20:41:12.268811563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4b0eaddb922ae234 immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="885 kB" avg_uncomp="885 kB" full=1 labels="{stream=\"stat-rep-ldla\"}"
level=error ts=2026-04-29T20:41:12.272259781Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4b0eaddb922ae234/19ddab8ac16%3A19ddac127f7%3Aacbba898?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-ldla\"}"
level=info ts=2026-04-29T20:41:13.088104052Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:41:13.088516296Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=warn ts=2026-04-29T20:41:13.109841003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:41:13.109844504Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:41:13.109868208Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:41:13.109876181Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:41:14.668550925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:41:14.675426213Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T20:41:15.361298675Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
ts=2026-04-29T20:41:16.375501291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=warn ts=2026-04-29T20:41:18.110348581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:41:18.203576183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:41:18.207344923Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-29T20:41:18.73605116Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-04-29T20:41:18.832323162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:41:18.83522163Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-29T20:41:20.505591307Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:41:21.363787426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:41:21.366990745Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-29T20:41:21.440221365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:41:21.446733544Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:41:21.860580916Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T20:41:22.11271083Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-04-29T20:41:22.112733099Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=7
level=info ts=2026-04-29T20:41:22.39948775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="658 kB" avg_comp="658 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:41:22.402787342Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-29T20:41:23.109893514Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:41:24.027481353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:41:24.030500632Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-29T20:41:25.486695952Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=info ts=2026-04-29T20:41:26.475266624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:41:26.478033833Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-29T20:41:28.110353972Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:41:28.861751317Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
level=info ts=2026-04-29T20:41:29.330192059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:41:29.338234805Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-29T20:41:30.237135063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:36980"
level=info ts=2026-04-29T20:41:30.803197406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:41:30.806621396Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-29T20:41:30.968042244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:41:30.970819051Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-29T20:41:32.236609949Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-04-29T20:41:32.532488483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:41:32.538539711Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T20:41:33.110538918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:41:35.611301126Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
ts=2026-04-29T20:41:37.006070644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:47506"
level=warn ts=2026-04-29T20:41:38.110377779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:41:38.986056392Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-04-29T20:41:39.062224813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:41:39.06742201Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-29T20:41:39.333458903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:41:39.337128429Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:41:39.337150401Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:41:39.337172325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:41:39.340419943Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-29T20:41:42.356153686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:41:42.359937045Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-29T20:41:42.362666171Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-04-29T20:41:42.362681424Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=6
level=info ts=2026-04-29T20:41:43.110467907Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:41:43.110505036Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:41:43.110515512Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:41:43.110546159Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T20:41:44.037778632Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:38852"
level=info ts=2026-04-29T20:41:45.02914455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:41:45.032013677Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-29T20:41:45.736039152Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-04-29T20:41:46.115316863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:41:46.11808494Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-04-29T20:41:46.378653026Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=warn ts=2026-04-29T20:41:48.110419611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:41:49.112701848Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-04-29T20:41:49.112724685Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=2
level=info ts=2026-04-29T20:41:49.875721402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:41:49.899687223Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-29T20:41:50.261634462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:41:50.265596849Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:41:51.593005782Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:41:52.486609079Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-29T20:41:52.486631123Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=warn ts=2026-04-29T20:41:53.109951709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:41:55.860654344Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-04-29T20:41:56.466314662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4b0eaddb922ae234 immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="885 kB" avg_uncomp="885 kB" full=1 labels="{stream=\"stat-rep-ldla\"}"
level=error ts=2026-04-29T20:41:56.469884244Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4b0eaddb922ae234/19ddab8ac16%3A19ddac127f7%3Aacbba898?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-ldla\"}"
level=info ts=2026-04-29T20:41:57.473339117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:41:57.476254322Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-29T20:41:58.110820724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:41:58.494486005Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T20:41:59.236430275Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
level=info ts=2026-04-29T20:41:59.805349624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:41:59.812518378Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:41:59.812538861Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:41:59.812561994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:41:59.819142347Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-29T20:42:00.262455331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:42:00.268852702Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T20:42:02.611145556Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-04-29T20:42:03.110893844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:42:03.298429888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.0 MB" avg_comp="1.0 MB" total_uncomp="16 MB" avg_uncomp="8.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:42:03.302032676Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T20:42:03.313064668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:42:03.315968044Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-29T20:42:04.80088087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="944 B" avg_comp="944 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:42:04.803535726Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-29T20:42:05.9866702Z caller=checkpoint.go:372 msg="writing series" size="904 kB"
level=debug ts=2026-04-29T20:42:05.98669207Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-04-29T20:42:06.170678432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:42:06.178186341Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T20:42:08.111177556Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:42:09.361059139Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-04-29T20:42:11.120408186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:42:11.12406344Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-29T20:42:12.73655783Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-04-29T20:42:13.027870366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:42:13.030399855Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-29T20:42:13.087627324Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:42:13.088663894Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-29T20:42:13.10995822Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:42:13.109981623Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:42:13.109989455Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:42:13.110001321Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:42:13.126373111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:42:13.132958772Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:42:14.322128244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:42:14.325210685Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-29T20:42:14.991558821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:42:14.997460457Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-29T20:42:16.111218546Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
ts=2026-04-29T20:42:16.381810383Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=info ts=2026-04-29T20:42:16.653690499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:42:16.657271107Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-29T20:42:18.110375551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:42:19.486249501Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-04-29T20:42:22.861281224Z caller=checkpoint.go:372 msg="writing series" size="347 kB"
level=warn ts=2026-04-29T20:42:23.110421626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:42:23.523354873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:42:23.527711446Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-29T20:42:25.100316341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:42:25.107015609Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:42:26.236183797Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-04-29T20:42:26.9944559Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:42:27.630306995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:42:27.634238862Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-29T20:42:28.110015201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:42:28.236300984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:42:28.23935424Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-29T20:42:28.727453694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:42:28.731016363Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-29T20:42:29.611798398Z caller=checkpoint.go:372 msg="writing series" size="627 kB"
level=debug ts=2026-04-29T20:42:29.611817899Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=7
level=debug ts=2026-04-29T20:42:32.987194206Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-29T20:42:32.987215362Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=warn ts=2026-04-29T20:42:33.110217158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:42:36.208631064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:42:36.211777821Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-29T20:42:36.361283995Z caller=checkpoint.go:372 msg="writing series" size="262 kB"
ts=2026-04-29T20:42:37.010912492Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:54886"
level=warn ts=2026-04-29T20:42:38.110555705Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:42:38.807524395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:42:38.810731038Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-29T20:42:39.795064445Z caller=checkpoint.go:372 msg="writing series" size="59 MB"
level=debug ts=2026-04-29T20:42:39.795090611Z caller=checkpoint.go:384 msg="flushing series" totalSize="60 MB" series=2
level=info ts=2026-04-29T20:42:40.537160309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:42:40.539944342Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-29T20:42:40.591338603Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:42:40.593036049Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-29T20:42:40.874797872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:42:40.883334452Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-29T20:42:43.109782594Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:42:43.109818805Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:42:43.10982743Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:42:43.109893544Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T20:42:44.041650922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:55260"
level=info ts=2026-04-29T20:42:45.664951105Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000015.tmp new=/var/loki/wal/checkpoint.000015
level=info ts=2026-04-29T20:42:45.666259374Z caller=checkpoint.go:670 msg="checkpoint done" time=4m32.555703512s
ts=2026-04-29T20:42:46.384305888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=info ts=2026-04-29T20:42:47.824645045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:42:47.829451427Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-29T20:42:47.998960701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.0 MB" avg_comp="1.0 MB" total_uncomp="16 MB" avg_uncomp="8.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:42:48.002716497Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-29T20:42:48.110260071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:42:49.274301981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:42:49.277230127Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-29T20:42:51.317473366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:42:51.320249217Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-29T20:42:52.177669751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:42:52.185064072Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:42:52.185086258Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:42:52.185123305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:42:52.192200699Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T20:42:53.109898578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:42:53.633248829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4b0eaddb922ae234 immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="885 kB" avg_uncomp="885 kB" full=1 labels="{stream=\"stat-rep-ldla\"}"
level=error ts=2026-04-29T20:42:53.637523797Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4b0eaddb922ae234/19ddab8ac16%3A19ddac127f7%3Aacbba898?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-ldla\"}"
level=info ts=2026-04-29T20:42:54.171182148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:42:54.178248525Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T20:42:55.04780584Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:42:55.654781087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:42:55.65742358Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-29T20:42:56.122075665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:42:56.124502984Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-29T20:42:57.610651279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:42:57.617585697Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-29T20:42:58.110379528Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T20:43:00.2445801Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:43096"
level=warn ts=2026-04-29T20:43:03.110504666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:43:03.596749025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:43:03.599672949Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:43:05.456219731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:43:05.460232001Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-29T20:43:06.419752869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:43:06.427447159Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T20:43:06.809553065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:43:06.812350698Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-04-29T20:43:07.013388581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:38662"
level=warn ts=2026-04-29T20:43:08.110870315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:43:08.983705337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:43:08.987304653Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-29T20:43:11.601185281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:43:11.604583436Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:43:11.6046022Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:43:11.604620915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:43:11.607891823Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-29T20:43:11.972918504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:43:11.975886372Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-29T20:43:12.9544999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:43:12.958227466Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:43:13.08779164Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:43:13.088862347Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-29T20:43:13.110238187Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-29T20:43:13.110267606Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T20:43:13.110269863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:43:13.110295441Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:43:13.110306245Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:43:13.110451866Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000016
level=info ts=2026-04-29T20:43:13.110463584Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-29T20:43:13.110486504Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.858µs success=true
level=debug ts=2026-04-29T20:43:13.110745575Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
ts=2026-04-29T20:43:14.044680212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:48164"
ts=2026-04-29T20:43:16.387746342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T20:43:16.487087193Z caller=checkpoint.go:372 msg="writing series" size="294 kB"
level=warn ts=2026-04-29T20:43:18.110559768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:43:19.741842435Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T20:43:19.861102384Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-04-29T20:43:23.110503177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:43:23.236806391Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-04-29T20:43:23.682561006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:43:23.68781768Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-29T20:43:24.443849245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:43:24.451594102Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:43:24.957150162Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:43:26.611129228Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
level=info ts=2026-04-29T20:43:27.069230046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:43:27.072464258Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-04-29T20:43:28.110455142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:43:29.532109577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:43:29.535179946Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-29T20:43:29.986283134Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
ts=2026-04-29T20:43:30.248237594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:43810"
level=info ts=2026-04-29T20:43:31.185265905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:43:31.192618351Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-29T20:43:33.109902858Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:43:33.361478585Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=info ts=2026-04-29T20:43:35.862353825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:43:35.870126708Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:43:35.870150737Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:43:35.870173964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:43:35.87719629Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T20:43:36.736770425Z caller=checkpoint.go:372 msg="writing series" size="5.6 kB"
ts=2026-04-29T20:43:37.016062142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:39158"
level=warn ts=2026-04-29T20:43:38.110622005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:43:38.313239638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:43:38.315830933Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-29T20:43:39.686778612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="465 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:43:39.693902425Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T20:43:39.859485449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:43:39.863012819Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-29T20:43:40.111900964Z caller=checkpoint.go:372 msg="writing series" size="641 kB"
level=debug ts=2026-04-29T20:43:40.111922743Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=9
level=info ts=2026-04-29T20:43:41.154198468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.0 MB" avg_comp="1.0 MB" total_uncomp="16 MB" avg_uncomp="8.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:43:41.158248854Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T20:43:43.110303972Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:43:43.11033534Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:43:43.110343148Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:43:43.110351897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:43:43.490233097Z caller=checkpoint.go:372 msg="writing series" size="4.1 MB"
level=debug ts=2026-04-29T20:43:43.490254615Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.1 MB" series=1
level=info ts=2026-04-29T20:43:44.052694604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:43:44.055609229Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:43:44.360497271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="889 B" avg_comp="889 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:43:44.363337477Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:43:44.363359159Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:43:44.363380477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:43:44.365891716Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-29T20:43:45.964741351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:43:45.967555645Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-04-29T20:43:46.391308233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:43:46.861677651Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-04-29T20:43:46.868787262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:43:46.871773799Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-29T20:43:47.218976173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:43:47.226675019Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:43:47.433204049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:43:47.441234099Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=warn ts=2026-04-29T20:43:48.110233824Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:43:48.349960897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4b0eaddb922ae234 immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="885 kB" avg_uncomp="885 kB" full=1 labels="{stream=\"stat-rep-ldla\"}"
level=error ts=2026-04-29T20:43:48.354331058Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4b0eaddb922ae234/19ddab8ac16%3A19ddac127f7%3Aacbba898?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-ldla\"}"
level=debug ts=2026-04-29T20:43:50.235938788Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=warn ts=2026-04-29T20:43:53.110469081Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:43:53.515118531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:43:53.518086689Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-29T20:43:53.611963768Z caller=checkpoint.go:372 msg="writing series" size="549 kB"
level=info ts=2026-04-29T20:43:53.971487908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:43:53.978178805Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:43:53.978199845Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:43:53.978224181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:43:53.981561764Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-29T20:43:56.98584974Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-29T20:43:58.110010386Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:43:59.432498849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:43:59.436999518Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-29T20:43:59.729494732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:43:59.732477517Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-29T20:44:00.066083559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:44:00.070115617Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:44:00.199780995Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-29T20:44:00.251573395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:48444"
level=info ts=2026-04-29T20:44:00.353234824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:44:00.356121023Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-29T20:44:00.361257782Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=info ts=2026-04-29T20:44:02.637660479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:44:02.640962548Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-04-29T20:44:03.109904816Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:44:03.73634236Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-04-29T20:44:04.83400771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:44:04.837384372Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-29T20:44:06.081111467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:44:06.083577089Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-29T20:44:07.11188165Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=warn ts=2026-04-29T20:44:08.11025711Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:44:10.486015893Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=info ts=2026-04-29T20:44:11.57473702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:44:11.5788394Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-29T20:44:13.087898569Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:44:13.087915522Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:44:13.110329326Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T20:44:13.110341462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:44:13.110355464Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:44:13.110364122Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:44:13.661057327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:44:13.669618402Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:44:13.861368677Z caller=checkpoint.go:372 msg="writing series" size="217 kB"
level=debug ts=2026-04-29T20:44:13.861389875Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=info ts=2026-04-29T20:44:13.939628415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:44:13.946239339Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-04-29T20:44:16.39430901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:44:17.236601674Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=warn ts=2026-04-29T20:44:18.110291076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:44:19.113071133Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-29T20:44:20.347291826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="489 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:44:20.355478558Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:44:20.611667206Z caller=checkpoint.go:372 msg="writing series" size="532 kB"
level=info ts=2026-04-29T20:44:21.931722562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:44:21.934271203Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-04-29T20:44:23.110277952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:44:23.986323262Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=info ts=2026-04-29T20:44:24.448630796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:44:24.456737007Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T20:44:24.863250582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:44:24.86600903Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-29T20:44:27.36102535Z caller=checkpoint.go:372 msg="writing series" size="6.5 kB"
level=warn ts=2026-04-29T20:44:28.1100983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:44:28.921616539Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-29T20:44:30.255047411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:57942"
level=debug ts=2026-04-29T20:44:30.736504663Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-04-29T20:44:30.752629148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:44:30.755545037Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-29T20:44:31.773291011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4b0eaddb922ae234 immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="885 kB" avg_uncomp="885 kB" full=1 labels="{stream=\"stat-rep-ldla\"}"
level=error ts=2026-04-29T20:44:31.776268783Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4b0eaddb922ae234/19ddab8ac16%3A19ddac127f7%3Aacbba898?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-ldla\"}"
level=warn ts=2026-04-29T20:44:33.110496606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:44:34.111954312Z caller=checkpoint.go:372 msg="writing series" size="550 kB"
level=debug ts=2026-04-29T20:44:34.111980639Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=debug ts=2026-04-29T20:44:37.486533619Z caller=checkpoint.go:372 msg="writing series" size="642 kB"
level=info ts=2026-04-29T20:44:38.097481491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:44:38.101050209Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-29T20:44:38.110126925Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:44:38.856225219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.0 MB" avg_comp="1.0 MB" total_uncomp="16 MB" avg_uncomp="8.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:44:38.860547415Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T20:44:39.393631218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:44:39.396419471Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-29T20:44:40.078935789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:44:40.081967014Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-29T20:44:40.861067051Z caller=checkpoint.go:372 msg="writing series" size="245 kB"
level=info ts=2026-04-29T20:44:41.484130822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:44:41.490769292Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:44:42.098572053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:44:42.102296729Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:44:43.051613345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:44:43.055949233Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-29T20:44:43.109853624Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:44:43.109885082Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-29T20:44:43.109870789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:44:43.109893667Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:44:44.193037599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:44:44.196493451Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-29T20:44:44.23662175Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-04-29T20:44:45.170940464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:44:45.174768187Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-29T20:44:45.793645789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:44:45.796084834Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-04-29T20:44:46.396786749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:44:47.612357786Z caller=checkpoint.go:372 msg="writing series" size="761 kB"
level=debug ts=2026-04-29T20:44:47.612380196Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=4
level=info ts=2026-04-29T20:44:47.944799684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:44:47.947845553Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-04-29T20:44:48.110844815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:44:50.986789726Z caller=checkpoint.go:372 msg="writing series" size="280 kB"
level=warn ts=2026-04-29T20:44:53.110764297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:44:53.653935993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:44:53.658906932Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-29T20:44:54.361016893Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-04-29T20:44:56.146857493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:44:56.151793442Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-29T20:44:57.736611669Z caller=checkpoint.go:372 msg="writing series" size="745 kB"
level=info ts=2026-04-29T20:44:57.738284213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:44:57.741965193Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-29T20:44:58.110768155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:45:01.11153603Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
level=debug ts=2026-04-29T20:45:01.111558142Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=debug ts=2026-04-29T20:45:02.063632244Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-29T20:45:03.11082025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:45:04.486692851Z caller=checkpoint.go:372 msg="writing series" size="261 kB"
level=debug ts=2026-04-29T20:45:04.808950213Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:45:04.837968852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:45:04.844543985Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-29T20:45:05.205012095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="489 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:45:05.211777105Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-29T20:45:07.02204066Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:52014"
level=debug ts=2026-04-29T20:45:07.861077422Z caller=checkpoint.go:372 msg="writing series" size="653 B"
level=warn ts=2026-04-29T20:45:08.110578069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:45:11.23655216Z caller=checkpoint.go:372 msg="writing series" size="592 kB"
level=info ts=2026-04-29T20:45:11.324757817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:45:11.327684294Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-29T20:45:12.168264976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:45:12.174534099Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-29T20:45:12.708179552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:45:12.712147999Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:45:12.712169386Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:45:12.71219081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:45:12.714157648Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-29T20:45:13.087898527Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:45:13.088330297Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=debug ts=2026-04-29T20:45:13.089648118Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-29T20:45:13.089702519Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974994 n=1
level=debug ts=2026-04-29T20:45:13.089748907Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974994 n=1
level=info ts=2026-04-29T20:45:13.110716779Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T20:45:13.110717864Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:45:13.110737471Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:45:13.110756728Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:45:13.715112393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:45:13.721544898Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T20:45:14.113785641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:45:14.11640313Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=debug ts=2026-04-29T20:45:14.611461797Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-04-29T20:45:16.142969954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4b0eaddb922ae234 immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="885 kB" avg_uncomp="885 kB" full=1 labels="{stream=\"stat-rep-ldla\"}"
level=error ts=2026-04-29T20:45:16.147136446Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4b0eaddb922ae234/19ddab8ac16%3A19ddac127f7%3Aacbba898?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-ldla\"}"
ts=2026-04-29T20:45:16.399865578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T20:45:17.986810637Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=debug ts=2026-04-29T20:45:17.98683495Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-04-29T20:45:18.110025955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:45:18.982607694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.0 MB" avg_comp="1.0 MB" total_uncomp="16 MB" avg_uncomp="8.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:45:18.987524212Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:45:21.361718895Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=warn ts=2026-04-29T20:45:23.110194564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:45:24.611475993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:45:24.614662263Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-29T20:45:24.736696601Z caller=checkpoint.go:372 msg="writing series" size="8.7 kB"
level=info ts=2026-04-29T20:45:26.921620824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:45:26.925323347Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:45:26.925343808Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:45:26.925371699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:45:26.927706978Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-29T20:45:28.110089195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:45:28.12536499Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-04-29T20:45:28.125391002Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=3
level=info ts=2026-04-29T20:45:29.08199154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:45:29.085136877Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-29T20:45:29.527347737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:45:29.533756034Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:45:29.533778748Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:45:29.533804717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:45:29.539701556Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:45:29.624874626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:45:29.627697243Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-29T20:45:29.708867254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:45:29.711474796Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-29T20:45:30.27410812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:45:30.276826399Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:45:30.27684655Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:45:30.276867271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:45:30.278957991Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-29T20:45:30.795341659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:45:30.799326516Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-29T20:45:31.215630442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:45:31.219916275Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-04-29T20:45:31.219940186Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:45:31.219961472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:45:31.222150505Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-29T20:45:31.488535383Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-04-29T20:45:33.110249125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:45:34.862997485Z caller=checkpoint.go:372 msg="writing series" size="3.0 MB"
level=debug ts=2026-04-29T20:45:34.86301925Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=2
level=info ts=2026-04-29T20:45:36.905630391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:45:36.909351412Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-29T20:45:37.434818669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:45:37.438389847Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-29T20:45:37.926965589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:45:37.930311897Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-29T20:45:38.11087598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:45:38.236230426Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-04-29T20:45:38.79518784Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:45:38.795230666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=801f5e087572d469 immediate=false num_chunks=1 total_comp="287 kB" avg_comp="287 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" idle=1 labels="{stream=\"build-e2e-cwjf\"}"
level=error ts=2026-04-29T20:45:38.802208687Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/801f5e087572d469/19ddab80be5%3A19ddac62d16%3Aafbdc1db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-cwjf\"}"
level=info ts=2026-04-29T20:45:41.347117244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:45:41.350395826Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-29T20:45:41.611222542Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=info ts=2026-04-29T20:45:42.869621552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:45:42.872390689Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:45:43.110568324Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:45:43.11059566Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:45:43.110603094Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:45:43.110601728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:45:44.986899732Z caller=checkpoint.go:372 msg="writing series" size="637 kB"
level=info ts=2026-04-29T20:45:45.469358102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:45:45.471822253Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-04-29T20:45:46.402835096Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=info ts=2026-04-29T20:45:47.706989238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:45:47.710771078Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-29T20:45:48.110150119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:45:48.361125458Z caller=checkpoint.go:372 msg="writing series" size="141 kB"
level=info ts=2026-04-29T20:45:48.577993185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:45:48.585136026Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:45:48.684110811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:45:48.686594982Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-29T20:45:51.736777726Z caller=checkpoint.go:372 msg="writing series" size="276 kB"
level=debug ts=2026-04-29T20:45:51.736806428Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-04-29T20:45:52.880356724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:45:52.883250652Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:45:52.883270984Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:45:52.883292044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:45:52.885549866Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-04-29T20:45:53.11023688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:45:53.451782251Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-29T20:45:54.342627738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="489 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:45:54.350653218Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:45:55.111135671Z caller=checkpoint.go:372 msg="writing series" size="222 kB"
level=info ts=2026-04-29T20:45:55.446749886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=801f5e087572d469 immediate=false num_chunks=1 total_comp="307 kB" avg_comp="307 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"build-e2e-cwjf\"}"
level=error ts=2026-04-29T20:45:55.451066078Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/801f5e087572d469/19ddab80be5%3A19ddac62d16%3Aafbdc1db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-cwjf\"}"
level=info ts=2026-04-29T20:45:55.582622458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:45:55.585451813Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-29T20:45:57.506995193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:45:57.510296013Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-29T20:45:58.078991502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:45:58.086348708Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=warn ts=2026-04-29T20:45:58.110568953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:45:58.499151682Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-04-29T20:45:58.499173997Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=2
level=info ts=2026-04-29T20:46:00.360558182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:46:00.368046649Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:46:01.861076295Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-29T20:46:02.557495014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:46:02.56933842Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-29T20:46:03.110684355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:46:05.236005285Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=info ts=2026-04-29T20:46:05.385580918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:46:05.388476231Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:46:06.762312656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4b0eaddb922ae234 immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="885 kB" avg_uncomp="885 kB" full=1 labels="{stream=\"stat-rep-ldla\"}"
level=error ts=2026-04-29T20:46:06.765505039Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4b0eaddb922ae234/19ddab8ac16%3A19ddac127f7%3Aacbba898?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-ldla\"}"
level=warn ts=2026-04-29T20:46:08.110086969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:46:08.611037134Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-04-29T20:46:10.101026904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.0 MB" avg_comp="1.0 MB" total_uncomp="16 MB" avg_uncomp="8.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:46:10.104732252Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T20:46:11.138017984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:46:11.140891026Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-29T20:46:11.986475514Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-29T20:46:13.087974482Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:46:13.088207763Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:46:13.110563252Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:46:13.110593611Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:46:13.110600945Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:46:13.110621248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:46:13.7910577Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:46:13.803033433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:46:13.806667699Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-29T20:46:14.372483632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:46:14.374992141Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:46:14.375013644Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:46:14.375044122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:46:14.377158064Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-29T20:46:15.361212701Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
level=info ts=2026-04-29T20:46:16.386404581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:46:16.389138421Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-04-29T20:46:16.405110484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=warn ts=2026-04-29T20:46:18.109898126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:46:18.736762385Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-04-29T20:46:20.248809467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:46:20.253332745Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:46:20.25336014Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:46:20.2533915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:46:20.256601316Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-29T20:46:21.25602537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:46:21.258765434Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-29T20:46:21.576784061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:46:21.585468856Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T20:46:22.113023955Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-04-29T20:46:22.11304823Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=7
level=warn ts=2026-04-29T20:46:23.110112838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:46:23.410634761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:46:23.413060442Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:46:23.413080996Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:46:23.413100649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:46:23.415043404Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-29T20:46:25.486584734Z caller=checkpoint.go:372 msg="writing series" size="216 kB"
level=warn ts=2026-04-29T20:46:28.109906504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:46:28.230241175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:46:28.234576921Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-29T20:46:28.440855708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=801f5e087572d469 immediate=false num_chunks=1 total_comp="307 kB" avg_comp="307 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"build-e2e-cwjf\"}"
level=error ts=2026-04-29T20:46:28.443936071Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/801f5e087572d469/19ddab80be5%3A19ddac62d16%3Aafbdc1db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-cwjf\"}"
level=info ts=2026-04-29T20:46:28.532158667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:46:28.534839447Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:46:28.534858579Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:46:28.534878452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:46:28.536899736Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-29T20:46:28.861211448Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
level=info ts=2026-04-29T20:46:31.314488973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:46:31.317284285Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-29T20:46:32.236457091Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-04-29T20:46:33.110524408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:46:33.957844515Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:46:33.95948088Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-29T20:46:35.611509816Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=info ts=2026-04-29T20:46:36.036666799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:46:36.039689362Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-04-29T20:46:36.03971265Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:46:36.039736685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:46:36.042233877Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-29T20:46:36.527116809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:46:36.529758958Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-29T20:46:37.718428863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:46:37.720910993Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-29T20:46:38.109839351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:46:38.984116653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-29T20:46:38.985782242Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=error ts=2026-04-29T20:46:38.986707124Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:46:38.986727249Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:46:38.986749824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:46:38.988779954Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-29T20:46:39.213489427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:46:39.216788686Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-29T20:46:39.759232361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:46:39.762163391Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-29T20:46:42.252042104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:46:42.259063216Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T20:46:42.362268502Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-04-29T20:46:42.362292237Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=6
level=info ts=2026-04-29T20:46:42.980186906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:46:42.982699375Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-29T20:46:42.991763399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:46:42.997677092Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:46:42.997693596Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:46:42.997714355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:46:43.003816232Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-29T20:46:43.110312133Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:46:43.110341953Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:46:43.11034949Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:46:43.110353195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:46:45.367491791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:46:45.370204649Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-29T20:46:45.483744849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="489 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:46:45.490663986Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:46:45.736341405Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
ts=2026-04-29T20:46:46.408220331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=warn ts=2026-04-29T20:46:48.110347951Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:46:49.004457103Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:46:49.114174249Z caller=checkpoint.go:372 msg="writing series" size="3.7 MB"
level=debug ts=2026-04-29T20:46:49.114196302Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=2
level=info ts=2026-04-29T20:46:50.603298847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="944 B" avg_comp="944 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:46:50.606983975Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:46:51.451299488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4b0eaddb922ae234 immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="885 kB" avg_uncomp="885 kB" full=1 labels="{stream=\"stat-rep-ldla\"}"
level=error ts=2026-04-29T20:46:51.455156391Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4b0eaddb922ae234/19ddab8ac16%3A19ddac127f7%3Aacbba898?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-ldla\"}"
level=error ts=2026-04-29T20:46:51.455187911Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:46:51.45521956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="325 B" avg_comp="325 B" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-29T20:46:51.458313753Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19ddaad9c07%3A19ddac3cf9e%3Aed3cd4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-29T20:46:52.368907645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:46:52.372015192Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-29T20:46:52.487179596Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-29T20:46:52.487202677Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=warn ts=2026-04-29T20:46:53.11035991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:46:54.174973668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:46:54.178304082Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-29T20:46:54.955407811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:46:54.958687419Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-29T20:46:55.861637959Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
level=info ts=2026-04-29T20:46:56.761891096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.1 MB" avg_comp="1.0 MB" total_uncomp="16 MB" avg_uncomp="8.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:46:56.766031349Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:46:56.766053549Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:46:56.766076837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.1 MB" avg_comp="1.0 MB" total_uncomp="16 MB" avg_uncomp="8.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:46:56.769012423Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T20:46:57.369761348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:46:57.372324039Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-04-29T20:46:58.110455129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:46:59.23683404Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-04-29T20:46:59.298184085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:46:59.30622618Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-29T20:47:00.371749656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:47:00.379001845Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-29T20:47:02.611498792Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-04-29T20:47:03.110582772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:47:04.490833476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:47:04.494529598Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-29T20:47:05.232320435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:47:05.234971531Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-29T20:47:05.754471918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-29T20:47:05.7572079Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19ddaad9c07%3A19ddac3cf9e%3Aed3cd4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-29T20:47:05.986963903Z caller=checkpoint.go:372 msg="writing series" size="943 kB"
level=debug ts=2026-04-29T20:47:05.986985435Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-04-29T20:47:06.399342283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:47:06.402487142Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-29T20:47:06.906977809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:47:06.909480478Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-29T20:47:08.11011616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:47:08.857078101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:47:08.860567355Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-29T20:47:09.361154422Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-04-29T20:47:09.407929364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=801f5e087572d469 immediate=false num_chunks=1 total_comp="307 kB" avg_comp="307 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"build-e2e-cwjf\"}"
level=error ts=2026-04-29T20:47:09.410822077Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/801f5e087572d469/19ddab80be5%3A19ddac62d16%3Aafbdc1db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-cwjf\"}"
level=info ts=2026-04-29T20:47:12.328383856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:47:12.336221072Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T20:47:12.736418375Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-04-29T20:47:12.74047665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:47:12.743333198Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-29T20:47:13.088041771Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:47:13.088043255Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:47:13.110545623Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:47:13.110582772Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-29T20:47:13.110584405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:47:13.110603359Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:47:13.379337068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:47:13.382207763Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-29T20:47:13.818609835Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:47:14.836168449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.1 MB" avg_comp="1.0 MB" total_uncomp="16 MB" avg_uncomp="8.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:47:14.84004754Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:47:16.111221193Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
ts=2026-04-29T20:47:16.410872521Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=info ts=2026-04-29T20:47:16.691992005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:47:16.694744837Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-29T20:47:18.110854404Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:47:19.485940157Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-04-29T20:47:22.079188441Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-29T20:47:22.679910548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:47:22.688751159Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:47:22.861318415Z caller=checkpoint.go:372 msg="writing series" size="309 kB"
level=warn ts=2026-04-29T20:47:23.110362157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:47:24.909481027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:47:24.917847869Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T20:47:25.139714941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:47:25.143501563Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-29T20:47:26.020881391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="489 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:47:26.030420943Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:47:26.236704128Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-04-29T20:47:27.006983172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-29T20:47:27.009584864Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19ddaad9c07%3A19ddac3cf9e%3Aed3cd4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-04-29T20:47:28.110372956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:47:29.612341963Z caller=checkpoint.go:372 msg="writing series" size="627 kB"
level=debug ts=2026-04-29T20:47:29.612359552Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=7
level=info ts=2026-04-29T20:47:32.955793254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:47:32.959123509Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-29T20:47:32.987070249Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-29T20:47:32.987090157Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=1
level=warn ts=2026-04-29T20:47:33.109904059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:47:34.85319765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:47:34.862593824Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:47:36.361759714Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
ts=2026-04-29T20:47:37.03090971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:56724"
level=warn ts=2026-04-29T20:47:38.110381281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:47:39.79579488Z caller=checkpoint.go:372 msg="writing series" size="60 MB"
level=debug ts=2026-04-29T20:47:39.79582115Z caller=checkpoint.go:384 msg="flushing series" totalSize="60 MB" series=2
level=info ts=2026-04-29T20:47:42.024642777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:47:42.027790354Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:47:42.853389158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:47:42.856660075Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-29T20:47:43.110702682Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:47:43.110734408Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:47:43.110743509Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:47:43.110797466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:47:43.121070875Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000016.tmp new=/var/loki/wal/checkpoint.000016
level=info ts=2026-04-29T20:47:43.122391411Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.011722985s
ts=2026-04-29T20:47:44.065271104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:57814"
level=info ts=2026-04-29T20:47:44.356975727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:47:44.359966482Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-29T20:47:45.016383181Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-29T20:47:46.413938516Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=info ts=2026-04-29T20:47:47.479350383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:47:47.482449391Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-29T20:47:48.110224262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:47:48.743166747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:47:48.746454083Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-29T20:47:49.191813228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:47:49.195221811Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-29T20:47:50.001126906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.1 MB" avg_comp="1.0 MB" total_uncomp="16 MB" avg_uncomp="8.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:47:50.004703016Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T20:47:51.777582909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:47:51.785054918Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-29T20:47:53.110188585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:47:54.663710359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=801f5e087572d469 immediate=false num_chunks=1 total_comp="307 kB" avg_comp="307 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"build-e2e-cwjf\"}"
level=error ts=2026-04-29T20:47:54.667792416Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/801f5e087572d469/19ddab80be5%3A19ddac62d16%3Aafbdc1db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-cwjf\"}"
level=warn ts=2026-04-29T20:47:58.109900642Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T20:48:00.271599311Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:48142"
level=info ts=2026-04-29T20:48:02.344972398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:48:02.352073756Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:48:02.883682974Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-29T20:48:03.110750914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:48:03.385566808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:48:03.390671162Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-29T20:48:03.692380895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:48:03.698853586Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-29T20:48:06.416671165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:48:06.42011685Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-29T20:48:06.931625015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:48:06.934530064Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-29T20:48:07.227480454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-29T20:48:07.230382087Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19ddaad9c07%3A19ddac3cf9e%3Aed3cd4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-29T20:48:07.985647725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:48:07.989547066Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-29T20:48:08.11009876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:48:09.365569147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:48:09.372782655Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:48:11.648767025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:48:11.652690844Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-29T20:48:13.087960515Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:48:13.087983302Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:48:13.109422475Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-29T20:48:13.109640859Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000017
level=info ts=2026-04-29T20:48:13.109651048Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-29T20:48:13.109672317Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.298µs success=true
level=info ts=2026-04-29T20:48:13.109859521Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:48:13.109886874Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:48:13.109896524Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:48:13.109935596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:48:13.109978795Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
ts=2026-04-29T20:48:16.417428317Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T20:48:16.485551919Z caller=checkpoint.go:372 msg="writing series" size="294 kB"
level=debug ts=2026-04-29T20:48:17.223619312Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:48:17.421356499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="489 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:48:17.428203867Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T20:48:18.110205377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:48:19.264420695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:48:19.267464372Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-29T20:48:19.860544927Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-04-29T20:48:23.11077772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:48:23.235172171Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-04-29T20:48:24.395082752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:48:24.403136122Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T20:48:25.115945266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:48:25.132669868Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-29T20:48:25.583623793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:48:25.587918786Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:48:25.58794643Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:48:25.587970057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:48:25.591506276Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:48:26.610730169Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=info ts=2026-04-29T20:48:27.965554554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:48:27.968209193Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-29T20:48:28.109853717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:48:29.177687604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:48:29.180695658Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:48:29.756061973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:48:29.75878475Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-29T20:48:29.985469862Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=info ts=2026-04-29T20:48:30.08613097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:48:30.088727782Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-04-29T20:48:33.109840519Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:48:33.360373683Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=debug ts=2026-04-29T20:48:36.735798152Z caller=checkpoint.go:372 msg="writing series" size="5.8 kB"
ts=2026-04-29T20:48:37.036203526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:47988"
level=info ts=2026-04-29T20:48:37.550153353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:48:37.555798059Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-29T20:48:38.110108796Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:48:38.939967504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=801f5e087572d469 immediate=false num_chunks=1 total_comp="307 kB" avg_comp="307 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"build-e2e-cwjf\"}"
level=error ts=2026-04-29T20:48:38.943314261Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/801f5e087572d469/19ddab80be5%3A19ddac62d16%3Aafbdc1db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-cwjf\"}"
level=debug ts=2026-04-29T20:48:40.110850106Z caller=checkpoint.go:372 msg="writing series" size="651 kB"
level=debug ts=2026-04-29T20:48:40.110875534Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=9
level=info ts=2026-04-29T20:48:43.007735798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.1 MB" avg_comp="1.0 MB" total_uncomp="16 MB" avg_uncomp="8.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:48:43.01231733Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T20:48:43.110455813Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:48:43.110487615Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:48:43.110498994Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:48:43.11052613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:48:43.489858113Z caller=checkpoint.go:372 msg="writing series" size="4.1 MB"
level=debug ts=2026-04-29T20:48:43.489879137Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.1 MB" series=1
level=info ts=2026-04-29T20:48:43.766686077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:48:43.770735136Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-04-29T20:48:44.069533463Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:37422"
level=info ts=2026-04-29T20:48:45.393649144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:48:45.400762627Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:48:45.400787218Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:48:45.400813561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:48:45.407550327Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-29T20:48:46.310855337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="889 B" avg_comp="889 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:48:46.313674348Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
ts=2026-04-29T20:48:46.42116399Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T20:48:46.860940248Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-04-29T20:48:48.11012945Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:48:50.078479048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:48:50.081481955Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-29T20:48:50.100843642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:48:50.103952254Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-29T20:48:50.233207674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=debug ts=2026-04-29T20:48:50.235055362Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=error ts=2026-04-29T20:48:50.236983844Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-29T20:48:50.395694081Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:48:51.927635418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:48:51.931451773Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-04-29T20:48:53.110482107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:48:53.611250791Z caller=checkpoint.go:372 msg="writing series" size="586 kB"
level=debug ts=2026-04-29T20:48:56.985311998Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-04-29T20:48:57.204307502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:48:57.207354756Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-29T20:48:58.110769663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:48:58.678475503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:48:58.685761739Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:49:00.36019116Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=info ts=2026-04-29T20:49:00.581977487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:49:00.589708706Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-29T20:49:01.71299673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:49:01.720210474Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:49:02.455704934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-29T20:49:02.458605851Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19ddaad9c07%3A19ddac3cf9e%3Aed3cd4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-04-29T20:49:03.110450335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:49:03.735517774Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-04-29T20:49:03.982852535Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T20:49:07.038639551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:57292"
level=debug ts=2026-04-29T20:49:07.111148148Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=warn ts=2026-04-29T20:49:08.10997277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:49:09.381836578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:49:09.391143467Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-29T20:49:09.495163977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:49:09.498743615Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-29T20:49:10.485741497Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=info ts=2026-04-29T20:49:11.578933964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:49:11.581963859Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-29T20:49:13.087957555Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:49:13.087971353Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:49:13.11039296Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T20:49:13.11039288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:49:13.110418943Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:49:13.110427986Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-29T20:49:13.860756615Z caller=checkpoint.go:372 msg="writing series" size="313 kB"
level=debug ts=2026-04-29T20:49:13.860785177Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=9
ts=2026-04-29T20:49:16.424332844Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=info ts=2026-04-29T20:49:17.23033508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="489 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:49:17.23504462Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=error ts=2026-04-29T20:49:17.238090849Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:49:17.238109518Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:49:17.23814612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="489 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:49:17.244688371Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T20:49:17.899687998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:49:17.902267242Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-29T20:49:18.11067917Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:49:20.253329089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:49:20.257621791Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-29T20:49:20.321002599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:49:20.32798115Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T20:49:20.610936304Z caller=checkpoint.go:372 msg="writing series" size="532 kB"
level=debug ts=2026-04-29T20:49:21.715361867Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T20:49:23.109950083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:49:23.215264919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=801f5e087572d469 immediate=false num_chunks=1 total_comp="307 kB" avg_comp="307 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"build-e2e-cwjf\"}"
level=error ts=2026-04-29T20:49:23.21873223Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/801f5e087572d469/19ddab80be5%3A19ddac62d16%3Aafbdc1db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-cwjf\"}"
level=debug ts=2026-04-29T20:49:23.985275413Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=info ts=2026-04-29T20:49:24.497002095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:49:24.499711065Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-29T20:49:26.595820926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:49:26.598682596Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-29T20:49:27.3609288Z caller=checkpoint.go:372 msg="writing series" size="6.5 kB"
level=info ts=2026-04-29T20:49:27.862364092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:49:27.865376636Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-29T20:49:28.110417836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:49:28.847198986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:49:28.850826293Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:49:29.021391307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="489 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:49:29.028023935Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T20:49:30.275186103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:49:30.278459411Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-04-29T20:49:30.279913996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:46130"
level=debug ts=2026-04-29T20:49:30.735726322Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-04-29T20:49:31.925270114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:49:31.931461317Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-29T20:49:33.110287381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:49:33.631508709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:49:33.635002845Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-29T20:49:34.111010741Z caller=checkpoint.go:372 msg="writing series" size="652 kB"
level=debug ts=2026-04-29T20:49:34.111035052Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=info ts=2026-04-29T20:49:34.239949352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.1 MB" avg_comp="1.0 MB" total_uncomp="16 MB" avg_uncomp="8.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:49:34.243688518Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:49:37.485489724Z caller=checkpoint.go:372 msg="writing series" size="518 kB"
level=warn ts=2026-04-29T20:49:38.110072007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:49:38.981494236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:49:38.984376658Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=debug ts=2026-04-29T20:49:40.860831925Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=info ts=2026-04-29T20:49:42.841747478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-29T20:49:42.846475686Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19ddaad9c07%3A19ddac3cf9e%3Aed3cd4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-29T20:49:43.110318726Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:49:43.110349659Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:49:43.110358913Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:49:43.110363555Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:49:43.217842346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:49:43.221476955Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-29T20:49:44.235173471Z caller=checkpoint.go:372 msg="writing series" size="147 kB"
level=info ts=2026-04-29T20:49:44.336481296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:49:44.340507884Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:49:44.340530419Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:49:44.34055385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:49:44.343662017Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
ts=2026-04-29T20:49:46.427695669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T20:49:47.61101102Z caller=checkpoint.go:372 msg="writing series" size="761 kB"
level=debug ts=2026-04-29T20:49:47.611037973Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=4
level=warn ts=2026-04-29T20:49:48.110237876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:49:48.51357859Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:49:48.513628938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=186402ce1bc8ee9d immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="898 kB" avg_uncomp="898 kB" idle=1 labels="{stream=\"integration2-hlit\"}"
level=error ts=2026-04-29T20:49:48.519694187Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/186402ce1bc8ee9d/19ddac5b409%3A19ddac9e257%3Af9a45c27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-hlit\"}"
level=debug ts=2026-04-29T20:49:50.985985305Z caller=checkpoint.go:372 msg="writing series" size="280 kB"
level=debug ts=2026-04-29T20:49:51.417893949Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-29T20:49:51.48110549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:49:51.483984646Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-29T20:49:53.109843442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:49:54.360190268Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-04-29T20:49:54.908892296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:49:54.915604627Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-29T20:49:56.335729339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:49:56.342189311Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:49:56.877671159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:49:56.881016686Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-29T20:49:57.735908126Z caller=checkpoint.go:372 msg="writing series" size="745 kB"
level=warn ts=2026-04-29T20:49:58.110401357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:49:59.785393573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=186402ce1bc8ee9d immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="898 kB" avg_uncomp="898 kB" full=1 labels="{stream=\"integration2-hlit\"}"
level=error ts=2026-04-29T20:49:59.788393856Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/186402ce1bc8ee9d/19ddac5b409%3A19ddac9e257%3Af9a45c27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-hlit\"}"
level=info ts=2026-04-29T20:49:59.845555852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:49:59.847845917Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-04-29T20:50:00.283592182Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:35098"
level=info ts=2026-04-29T20:50:00.929838417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="489 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:50:00.936481741Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:50:01.110335712Z caller=checkpoint.go:372 msg="writing series" size="133 kB"
level=debug ts=2026-04-29T20:50:01.110359611Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-04-29T20:50:02.769765675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:50:02.773700046Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-04-29T20:50:03.110026871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:50:04.485426935Z caller=checkpoint.go:372 msg="writing series" size="291 kB"
level=info ts=2026-04-29T20:50:05.177706456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:50:05.180558392Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-29T20:50:06.842988614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:50:06.84599121Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-29T20:50:07.859944856Z caller=checkpoint.go:372 msg="writing series" size="653 B"
level=warn ts=2026-04-29T20:50:08.110893522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:50:08.980634436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:50:08.989168349Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:50:08.989187626Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:50:08.989214425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:50:08.995967806Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T20:50:11.236253176Z caller=checkpoint.go:372 msg="writing series" size="592 kB"
level=info ts=2026-04-29T20:50:11.640872519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:50:11.643651487Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-29T20:50:13.088091185Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:50:13.088101341Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:50:13.110530128Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:50:13.110559003Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-29T20:50:13.110554732Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:50:13.11056651Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:50:13.909950688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:50:13.917595654Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-04-29T20:50:14.076093217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:43386"
level=debug ts=2026-04-29T20:50:14.610078504Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-04-29T20:50:14.61010072Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-04-29T20:50:15.10491922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:50:15.107679094Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-29T20:50:15.313902169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:50:15.316806469Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:50:15.761000728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:50:15.765255303Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:50:15.860610236Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:50:15.948943285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=801f5e087572d469 immediate=false num_chunks=1 total_comp="307 kB" avg_comp="307 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"build-e2e-cwjf\"}"
level=error ts=2026-04-29T20:50:15.952034437Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/801f5e087572d469/19ddab80be5%3A19ddac62d16%3Aafbdc1db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-cwjf\"}"
ts=2026-04-29T20:50:16.430410353Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:50:17.985399092Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-04-29T20:50:18.110071745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:50:19.341405321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:50:19.348285683Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T20:50:21.360631229Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=info ts=2026-04-29T20:50:21.983386172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:50:21.98636342Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-29T20:50:22.285465506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:50:22.288717279Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-29T20:50:23.110758102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:50:24.735448417Z caller=checkpoint.go:372 msg="writing series" size="9.3 kB"
level=info ts=2026-04-29T20:50:25.118855494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:50:25.122149167Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-04-29T20:50:28.110777925Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:50:28.124704339Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-04-29T20:50:28.12472474Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=4
level=info ts=2026-04-29T20:50:29.164032088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:50:29.168609243Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:50:29.16863296Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:50:29.168656784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:50:29.171049656Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-29T20:50:29.546999956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:50:29.551572295Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
ts=2026-04-29T20:50:30.286873261Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:48036"
level=info ts=2026-04-29T20:50:30.554085035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:50:30.558489155Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:50:30.5585095Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:50:30.558532383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:50:30.561536418Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-29T20:50:31.363899672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-29T20:50:31.366915636Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19ddaad9c07%3A19ddac3cf9e%3Aed3cd4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-29T20:50:31.485479214Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-04-29T20:50:33.110446042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:50:33.224763673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.1 MB" avg_comp="1.0 MB" total_uncomp="16 MB" avg_uncomp="8.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:50:33.228858703Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T20:50:34.275272304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=186402ce1bc8ee9d immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="898 kB" avg_uncomp="898 kB" full=1 labels="{stream=\"integration2-hlit\"}"
level=error ts=2026-04-29T20:50:34.278407262Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/186402ce1bc8ee9d/19ddac5b409%3A19ddac9e257%3Af9a45c27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-hlit\"}"
level=debug ts=2026-04-29T20:50:34.862963408Z caller=checkpoint.go:372 msg="writing series" size="2.9 MB"
level=debug ts=2026-04-29T20:50:34.862990555Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=2
level=warn ts=2026-04-29T20:50:38.110276486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:50:38.235645175Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-04-29T20:50:38.429478484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:50:38.432291361Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-29T20:50:41.610612457Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=info ts=2026-04-29T20:50:41.694943728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:50:41.697648011Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=debug ts=2026-04-29T20:50:42.256688626Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:50:42.258278359Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-29T20:50:43.10980195Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:50:43.109857393Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-29T20:50:43.109853875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:50:43.109867433Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:50:43.309577615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:50:43.316851074Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:50:44.98571448Z caller=checkpoint.go:372 msg="writing series" size="703 kB"
level=info ts=2026-04-29T20:50:45.521147945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:50:45.529721827Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-29T20:50:45.598037698Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:50:45.710400835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="489 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:50:45.717316599Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-29T20:50:46.433515057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=warn ts=2026-04-29T20:50:48.110693311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:50:48.251997215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:50:48.256048204Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-29T20:50:48.360605942Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=info ts=2026-04-29T20:50:49.08967284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:50:49.096922516Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T20:50:49.401641499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:50:49.405179249Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-29T20:50:51.735194867Z caller=checkpoint.go:372 msg="writing series" size="309 kB"
level=debug ts=2026-04-29T20:50:51.735218815Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=warn ts=2026-04-29T20:50:53.110574186Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:50:55.110346723Z caller=checkpoint.go:372 msg="writing series" size="284 kB"
level=info ts=2026-04-29T20:50:55.577479888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:50:55.584319874Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-29T20:50:56.385718647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:50:56.388694351Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-29T20:50:58.110662667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:50:58.272018252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:50:58.275108812Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-29T20:50:58.498321383Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-04-29T20:50:58.498343619Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=2
level=info ts=2026-04-29T20:50:59.993707254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:50:59.996995996Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-04-29T20:51:00.290600146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:54582"
level=debug ts=2026-04-29T20:51:01.860857919Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-29T20:51:02.044487858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=801f5e087572d469 immediate=false num_chunks=1 total_comp="307 kB" avg_comp="307 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"build-e2e-cwjf\"}"
level=error ts=2026-04-29T20:51:02.047999142Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/801f5e087572d469/19ddab80be5%3A19ddac62d16%3Aafbdc1db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-cwjf\"}"
level=info ts=2026-04-29T20:51:02.47036208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:51:02.473383766Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-29T20:51:03.110874325Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:51:04.547019983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:51:04.550044034Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-29T20:51:05.235653039Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=info ts=2026-04-29T20:51:06.684070897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:51:06.686833784Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:51:07.433834839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:51:07.43795626Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-29T20:51:07.957644304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:51:07.960607136Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-04-29T20:51:08.110701974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:51:08.350144155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:51:08.353633234Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-29T20:51:08.610165549Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=debug ts=2026-04-29T20:51:11.985713626Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-29T20:51:13.088046367Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:51:13.088060982Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:51:13.110474549Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:51:13.110495651Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:51:13.110504462Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:51:13.110520874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-29T20:51:14.07980812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:59442"
level=debug ts=2026-04-29T20:51:15.360829922Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
ts=2026-04-29T20:51:16.437159483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=debug ts=2026-04-29T20:51:17.427687361Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:51:17.598202809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:51:17.603012213Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-04-29T20:51:18.110627585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:51:18.735955234Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-04-29T20:51:20.772823099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:51:20.775503639Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-29T20:51:20.870727243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=186402ce1bc8ee9d immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="898 kB" avg_uncomp="898 kB" full=1 labels="{stream=\"integration2-hlit\"}"
level=error ts=2026-04-29T20:51:20.873546982Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/186402ce1bc8ee9d/19ddac5b409%3A19ddac9e257%3Af9a45c27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-hlit\"}"
level=debug ts=2026-04-29T20:51:22.112222406Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-04-29T20:51:22.112247276Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=7
level=debug ts=2026-04-29T20:51:22.169703026Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-29T20:51:23.110711055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:51:23.364309269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:51:23.368149635Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-29T20:51:24.157116748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:51:24.159885208Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-29T20:51:24.826703573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:51:24.829473885Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-29T20:51:25.48604197Z caller=checkpoint.go:372 msg="writing series" size="241 kB"
level=info ts=2026-04-29T20:51:25.986231544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.1 MB" avg_comp="1.0 MB" total_uncomp="16 MB" avg_uncomp="8.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:51:25.989899942Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T20:51:27.080770059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="489 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:51:27.08714272Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T20:51:27.819319059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-29T20:51:27.822351988Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19ddaad9c07%3A19ddac3cf9e%3Aed3cd4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-04-29T20:51:28.110607887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:51:28.860679899Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=info ts=2026-04-29T20:51:30.914708373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:51:30.925232789Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-29T20:51:32.235114094Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-04-29T20:51:33.110629016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:51:35.610913391Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
ts=2026-04-29T20:51:37.051159291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:57574"
level=warn ts=2026-04-29T20:51:38.110190996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:51:38.985948182Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-04-29T20:51:40.153091856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:51:40.158067972Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:51:40.158091947Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:51:40.158114395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:51:40.160929571Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-29T20:51:41.621055604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:51:41.627414551Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:51:42.363319058Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-04-29T20:51:42.363342848Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=6
level=info ts=2026-04-29T20:51:43.109873766Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T20:51:43.109883616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:51:43.10990951Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:51:43.109920009Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:51:43.606979882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:51:43.613081651Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:51:43.835800314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:51:43.842202801Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-29T20:51:44.519276647Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:51:45.735735057Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
ts=2026-04-29T20:51:46.44017584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=info ts=2026-04-29T20:51:47.758132777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:51:47.767392082Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T20:51:47.959001657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="944 B" avg_comp="944 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:51:47.961371597Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:51:47.961390926Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:51:47.96141314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:51:47.963240815Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-29T20:51:48.110899269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:51:49.111789226Z caller=checkpoint.go:372 msg="writing series" size="3.7 MB"
level=debug ts=2026-04-29T20:51:49.111811941Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=2
level=info ts=2026-04-29T20:51:49.130655783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:51:49.133459864Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-29T20:51:51.115633437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:51:51.118933329Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-29T20:51:51.739147861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=801f5e087572d469 immediate=false num_chunks=1 total_comp="307 kB" avg_comp="307 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"build-e2e-cwjf\"}"
level=error ts=2026-04-29T20:51:51.742417252Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/801f5e087572d469/19ddab80be5%3A19ddac62d16%3Aafbdc1db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-cwjf\"}"
level=error ts=2026-04-29T20:51:51.742458552Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:51:51.742479128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=801f5e087572d469 immediate=false num_chunks=1 total_comp="307 kB" avg_comp="307 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"build-e2e-cwjf\"}"
level=error ts=2026-04-29T20:51:51.744918908Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/801f5e087572d469/19ddab80be5%3A19ddac62d16%3Aafbdc1db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-cwjf\"}"
level=debug ts=2026-04-29T20:51:52.486185757Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-29T20:51:52.486209544Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-04-29T20:51:52.492367834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:51:52.495224021Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-04-29T20:51:53.110637058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:51:54.431631626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:51:54.434598827Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-29T20:51:55.562045243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:51:55.564978846Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-29T20:51:55.860765606Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
level=info ts=2026-04-29T20:51:57.129835078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:51:57.132769443Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-04-29T20:51:57.132788118Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:51:57.132814387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:51:57.135081318Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-29T20:51:58.110648683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:51:59.235491383Z caller=checkpoint.go:372 msg="writing series" size="40 kB"
level=info ts=2026-04-29T20:51:59.885109671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:51:59.88810419Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-04-29T20:52:00.295289979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:53088"
level=debug ts=2026-04-29T20:52:02.611004948Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-04-29T20:52:03.110845453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:52:03.185130453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:52:03.188187617Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-29T20:52:04.272835796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:52:04.275892283Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-29T20:52:05.089178862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:52:05.093378462Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-29T20:52:05.343009547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:52:05.345979976Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=debug ts=2026-04-29T20:52:05.986481177Z caller=checkpoint.go:372 msg="writing series" size="959 kB"
level=debug ts=2026-04-29T20:52:05.98651007Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
ts=2026-04-29T20:52:07.054670239Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.81:41268"
level=info ts=2026-04-29T20:52:07.525348707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=801f5e087572d469 immediate=false num_chunks=1 total_comp="307 kB" avg_comp="307 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"build-e2e-cwjf\"}"
level=error ts=2026-04-29T20:52:07.528658064Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/801f5e087572d469/19ddab80be5%3A19ddac62d16%3Aafbdc1db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-cwjf\"}"
level=warn ts=2026-04-29T20:52:08.110351169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:52:09.061757635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:52:09.065098406Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-29T20:52:09.360220929Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-04-29T20:52:11.945696215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:52:11.950058808Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:52:12.735368259Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-04-29T20:52:13.088046509Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:52:13.08846954Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:52:13.109807645Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:52:13.109832434Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:52:13.109840338Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:52:13.109863767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:52:13.453374396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.1 MB" avg_comp="1.0 MB" total_uncomp="16 MB" avg_uncomp="8.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:52:13.456887882Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T20:52:13.664181029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:52:13.667620576Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-29T20:52:15.667625712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="489 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:52:15.675263253Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T20:52:15.701739989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=186402ce1bc8ee9d immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="898 kB" avg_uncomp="898 kB" full=1 labels="{stream=\"integration2-hlit\"}"
level=error ts=2026-04-29T20:52:15.704691711Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/186402ce1bc8ee9d/19ddac5b409%3A19ddac9e257%3Af9a45c27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-hlit\"}"
level=debug ts=2026-04-29T20:52:15.856154444Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-29T20:52:16.110379555Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
ts=2026-04-29T20:52:16.44306219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=warn ts=2026-04-29T20:52:18.110516407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:52:18.907460282Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:52:19.48564111Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-04-29T20:52:21.724488519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:52:21.73249473Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:52:22.860758043Z caller=checkpoint.go:372 msg="writing series" size="309 kB"
level=warn ts=2026-04-29T20:52:23.110224959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:52:23.592780763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:52:23.596354855Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-29T20:52:23.841055584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:52:23.843780644Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-29T20:52:24.537480045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:52:24.544846165Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:52:24.544868654Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:52:24.544895568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:52:24.552343593Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:52:25.091072106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:52:25.094081312Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:52:25.115624716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:52:25.118612256Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-29T20:52:25.614926609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:52:25.622168641Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-29T20:52:26.204053071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-29T20:52:26.207753142Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19ddaad9c07%3A19ddac3cf9e%3Aed3cd4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-29T20:52:26.235863114Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-04-29T20:52:28.110211778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:52:29.611040261Z caller=checkpoint.go:372 msg="writing series" size="627 kB"
level=debug ts=2026-04-29T20:52:29.611061317Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=7
level=info ts=2026-04-29T20:52:32.475353211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:52:32.482564706Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T20:52:32.986723541Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-29T20:52:32.986743639Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=1
level=warn ts=2026-04-29T20:52:33.110813291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:52:35.643370255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:52:35.646503359Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:52:35.646523312Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:52:35.646543932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:52:35.648708656Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-29T20:52:36.36048633Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=warn ts=2026-04-29T20:52:38.110423194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:52:38.325901641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:52:38.328815166Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:52:38.328837521Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:52:38.328859744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:52:38.331181067Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-29T20:52:39.793259751Z caller=checkpoint.go:372 msg="writing series" size="60 MB"
level=debug ts=2026-04-29T20:52:39.793285069Z caller=checkpoint.go:384 msg="flushing series" totalSize="60 MB" series=2
level=info ts=2026-04-29T20:52:39.80192752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:52:39.805419327Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:52:39.805458858Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:52:39.805480644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:52:39.808324973Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-29T20:52:42.885341149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:52:42.89581382Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:52:43.110657656Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:52:43.110698482Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:52:43.110707206Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:52:43.110705958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:52:43.121874542Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000017.tmp new=/var/loki/wal/checkpoint.000017
level=info ts=2026-04-29T20:52:43.12334832Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.013493053s
ts=2026-04-29T20:52:44.085944504Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:60892"
level=info ts=2026-04-29T20:52:44.69722754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:52:44.700111189Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-29T20:52:45.412229684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:52:45.41504971Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:52:45.415070639Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:52:45.4150907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:52:45.417340928Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-04-29T20:52:46.445884211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=info ts=2026-04-29T20:52:46.771163494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=801f5e087572d469 immediate=false num_chunks=1 total_comp="307 kB" avg_comp="307 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"build-e2e-cwjf\"}"
level=error ts=2026-04-29T20:52:46.774476126Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/801f5e087572d469/19ddab80be5%3A19ddac62d16%3Aafbdc1db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-cwjf\"}"
level=info ts=2026-04-29T20:52:47.739278494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:52:47.743658681Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-29T20:52:48.11085176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:52:50.557658449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:52:50.56169692Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-29T20:52:51.009562003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:52:51.012540945Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-04-29T20:52:53.109920913Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:52:53.61556583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:52:53.620823015Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:52:54.638275814Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:52:56.194667499Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-29T20:52:56.731301879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:52:56.734351035Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-29T20:52:58.110466748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:52:58.131597493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:52:58.134468196Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:52:58.134485699Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:52:58.134507836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:52:58.136912305Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-29T20:52:58.153986354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:52:58.156375287Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:52:58.156391945Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:52:58.435477697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:52:58.438419019Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-29T20:52:58.868637745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="489 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:52:58.877742268Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-29T20:52:59.435548261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:52:59.439148892Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-04-29T20:53:00.301311471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:42526"
level=warn ts=2026-04-29T20:53:03.110889689Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:53:04.151058782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:53:04.154054285Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-29T20:53:04.558127957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:53:04.561682291Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-29T20:53:06.985897339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=186402ce1bc8ee9d immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="898 kB" avg_uncomp="898 kB" full=1 labels="{stream=\"integration2-hlit\"}"
level=error ts=2026-04-29T20:53:06.989128875Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/186402ce1bc8ee9d/19ddac5b409%3A19ddac9e257%3Af9a45c27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-hlit\"}"
level=warn ts=2026-04-29T20:53:08.110361528Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:53:08.674734015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.1 MB" avg_comp="1.0 MB" total_uncomp="16 MB" avg_uncomp="8.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:53:08.67871934Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T20:53:08.701889658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:53:08.704172687Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-29T20:53:10.173298883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:53:10.176533903Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:53:10.881354927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-29T20:53:10.88423717Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19ddaad9c07%3A19ddac3cf9e%3Aed3cd4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-04-29T20:53:10.884255328Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:53:10.88427522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-29T20:53:10.886505064Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19ddaad9c07%3A19ddac3cf9e%3Aed3cd4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-29T20:53:10.951554649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:53:10.954134125Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-29T20:53:11.151014662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:53:11.153901211Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-29T20:53:11.66883705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:53:11.673012762Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-04-29T20:53:11.673036651Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:53:11.673058925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:53:11.676297886Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-29T20:53:12.228835031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:53:12.23151223Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-29T20:53:13.08798055Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:53:13.088012115Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:53:13.109414102Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-29T20:53:13.109578013Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000018
level=info ts=2026-04-29T20:53:13.10958662Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-29T20:53:13.109602614Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.377µs success=true
level=debug ts=2026-04-29T20:53:13.109820746Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-04-29T20:53:13.109815804Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:53:13.109837268Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:53:13.109849038Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:53:13.109863421Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:53:14.389628811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:53:14.397042761Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-29T20:53:16.449033401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a35146bd 10.128.0.81:7946"
level=debug ts=2026-04-29T20:53:16.486012514Z caller=checkpoint.go:372 msg="writing series" size="294 kB"
level=info ts=2026-04-29T20:53:16.859750685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:53:16.862610967Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:53:16.862641674Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:53:16.8626698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:53:16.864941139Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-04-29T20:53:18.110070538Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:53:19.689350351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:53:19.69630227Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-29T20:53:19.860211252Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-04-29T20:53:20.705993124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:53:20.713011252Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:53:20.713034898Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:53:20.713057634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:53:20.719869948Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-29T20:53:21.582411589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:53:21.585032275Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-29T20:53:22.722297559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:53:22.725910565Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-29T20:53:23.110335147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:53:23.235724927Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-04-29T20:53:23.778991601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:53:23.782300332Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-29T20:53:24.964713108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:53:24.967902003Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-29T20:53:26.610513091Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=info ts=2026-04-29T20:53:26.880190859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:53:26.882915412Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-04-29T20:53:28.11046669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:53:28.676936698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-29T20:53:28.679826637Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19ddaad9c07%3A19ddac3cf9e%3Aed3cd4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-29T20:53:29.682822558Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:53:29.682838621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5968eb640ba84b08 immediate=false num_chunks=1 total_comp="1.9 kB" avg_comp="1.9 kB" total_uncomp="189 kB" avg_uncomp="189 kB" idle=1 labels="{stream=\"build-e2e-cmho\"}"
level=error ts=2026-04-29T20:53:29.686583297Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5968eb640ba84b08/19ddab8e2bd%3A19ddacd31f6%3Ae4c1710c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-cmho\"}"
level=info ts=2026-04-29T20:53:29.983338149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3498cb8c2f07200d immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="188 kB" avg_uncomp="188 kB" idle=1 labels="{stream=\"build-e2e-dvis\"}"
level=debug ts=2026-04-29T20:53:29.985242696Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=error ts=2026-04-29T20:53:29.986852906Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3498cb8c2f07200d/19ddab85df6%3A19ddacd801f%3Af49f9823?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-dvis\"}"
level=info ts=2026-04-29T20:53:30.476112887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=801f5e087572d469 immediate=false num_chunks=1 total_comp="307 kB" avg_comp="307 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"build-e2e-cwjf\"}"
level=error ts=2026-04-29T20:53:30.479752489Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/801f5e087572d469/19ddab80be5%3A19ddac62d16%3Aafbdc1db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-cwjf\"}"
level=info ts=2026-04-29T20:53:31.645203738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:53:31.648010428Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-29T20:53:32.13962361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:53:32.147388013Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=warn ts=2026-04-29T20:53:33.110835298Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:53:33.360231915Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=info ts=2026-04-29T20:53:35.747269271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:53:35.754912351Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:53:36.735841886Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=warn ts=2026-04-29T20:53:38.109892882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:53:40.111359725Z caller=checkpoint.go:372 msg="writing series" size="666 kB"
level=debug ts=2026-04-29T20:53:40.111383139Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=9
level=info ts=2026-04-29T20:53:41.668931591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:53:41.673192563Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-29T20:53:42.394314373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:53:42.397737001Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-29T20:53:42.443789173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5968eb640ba84b08 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"build-e2e-cmho\"}"
level=error ts=2026-04-29T20:53:42.446573296Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5968eb640ba84b08/19ddab8e2bd%3A19ddacd31f6%3Ae4c1710c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-cmho\"}"
level=info ts=2026-04-29T20:53:43.110743653Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:53:43.110774033Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:53:43.110782671Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:53:43.110834689Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:53:43.488606735Z caller=checkpoint.go:372 msg="writing series" size="4.1 MB"
level=debug ts=2026-04-29T20:53:43.488636774Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.1 MB" series=1
ts=2026-04-29T20:53:44.090240438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:49698"
level=debug ts=2026-04-29T20:53:45.374176179Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-29T20:53:45.611800695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:53:45.615183802Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-04-29T20:53:46.451601931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=debug ts=2026-04-29T20:53:46.859999282Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-04-29T20:53:47.193324776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3498cb8c2f07200d immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-dvis\"}"
level=error ts=2026-04-29T20:53:47.196591515Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3498cb8c2f07200d/19ddab85df6%3A19ddacd801f%3Af49f9823?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-dvis\"}"
level=warn ts=2026-04-29T20:53:48.113544211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:53:49.43828676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.1 MB" avg_comp="1.0 MB" total_uncomp="16 MB" avg_uncomp="8.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:53:49.442139489Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:53:49.442163022Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:53:49.442187776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.1 MB" avg_comp="1.0 MB" total_uncomp="16 MB" avg_uncomp="8.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:53:49.445269479Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T20:53:49.479471075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:53:49.481965724Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-29T20:53:49.983151112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:53:49.986133416Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:53:50.234985713Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=warn ts=2026-04-29T20:53:53.109939979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:53:53.61092042Z caller=checkpoint.go:372 msg="writing series" size="606 kB"
level=info ts=2026-04-29T20:53:54.70059416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:53:54.706711637Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:53:54.943562975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=186402ce1bc8ee9d immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="898 kB" avg_uncomp="898 kB" full=1 labels="{stream=\"integration2-hlit\"}"
level=error ts=2026-04-29T20:53:54.946449693Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/186402ce1bc8ee9d/19ddac5b409%3A19ddac9e257%3Af9a45c27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-hlit\"}"
level=info ts=2026-04-29T20:53:55.374741178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:53:55.377532624Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-29T20:53:56.985220671Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-04-29T20:53:57.5956418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="489 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:53:57.602200485Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-29T20:53:58.110370631Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:53:59.597696803Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:54:00.012364136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:54:00.016311057Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-29T20:54:00.30636568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:32988"
level=debug ts=2026-04-29T20:54:00.360081862Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-04-29T20:54:00.413241091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:54:00.415747938Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-29T20:54:01.629321995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="889 B" avg_comp="889 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:54:01.631870002Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-29T20:54:01.871553004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-29T20:54:01.874195933Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19ddaad9c07%3A19ddac3cf9e%3Aed3cd4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-04-29T20:54:03.110345213Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:54:03.362671315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.1 MB" avg_comp="1.0 MB" total_uncomp="16 MB" avg_uncomp="8.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:54:03.366217518Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:54:03.735531073Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-04-29T20:54:04.032285022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:54:04.034952591Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-29T20:54:05.41632112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:54:05.422695627Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:54:05.591305409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:54:05.597366168Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-29T20:54:06.596201777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:54:06.602326986Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:54:07.110271834Z caller=checkpoint.go:372 msg="writing series" size="204 kB"
level=warn ts=2026-04-29T20:54:08.11056072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:54:08.342202726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:54:08.344570122Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-29T20:54:10.48487566Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=debug ts=2026-04-29T20:54:10.484899854Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-04-29T20:54:13.087916251Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:54:13.088301525Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:54:13.110640755Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:54:13.110674588Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:54:13.11068669Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:54:13.110674792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:54:13.196204408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3498cb8c2f07200d immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-dvis\"}"
level=error ts=2026-04-29T20:54:13.199159892Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3498cb8c2f07200d/19ddab85df6%3A19ddacd801f%3Af49f9823?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-dvis\"}"
level=debug ts=2026-04-29T20:54:13.860664245Z caller=checkpoint.go:372 msg="writing series" size="359 kB"
level=info ts=2026-04-29T20:54:15.909164744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:54:15.912306226Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-04-29T20:54:16.454845275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=info ts=2026-04-29T20:54:16.480577934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=801f5e087572d469 immediate=false num_chunks=1 total_comp="307 kB" avg_comp="307 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"build-e2e-cwjf\"}"
level=error ts=2026-04-29T20:54:16.483868139Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/801f5e087572d469/19ddab80be5%3A19ddac62d16%3Aafbdc1db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-cwjf\"}"
level=debug ts=2026-04-29T20:54:17.235840282Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=warn ts=2026-04-29T20:54:18.110615254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:54:18.389479332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5968eb640ba84b08 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"build-e2e-cmho\"}"
level=error ts=2026-04-29T20:54:18.39280571Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5968eb640ba84b08/19ddab8e2bd%3A19ddacd31f6%3Ae4c1710c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-cmho\"}"
level=info ts=2026-04-29T20:54:19.586574582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:54:19.58938137Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-29T20:54:20.610768391Z caller=checkpoint.go:372 msg="writing series" size="532 kB"
level=info ts=2026-04-29T20:54:21.961376246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:54:21.965235962Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-29T20:54:23.110478592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:54:23.984930045Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=info ts=2026-04-29T20:54:24.404204206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:54:24.407806395Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-29T20:54:25.931606335Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T20:54:25.933491946Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-29T20:54:27.252318522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:54:27.25529022Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-29T20:54:27.360674958Z caller=checkpoint.go:372 msg="writing series" size="6.5 kB"
level=warn ts=2026-04-29T20:54:28.110031714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:54:28.144193276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:54:28.150702744Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=debug ts=2026-04-29T20:54:28.258078305Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-29T20:54:30.73492619Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-29T20:54:33.110384443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:54:34.010717251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:54:34.014906113Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-29T20:54:34.1107889Z caller=checkpoint.go:372 msg="writing series" size="577 kB"
level=debug ts=2026-04-29T20:54:34.110812268Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=7
level=info ts=2026-04-29T20:54:34.901201745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:54:34.90409729Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-29T20:54:37.485067151Z caller=checkpoint.go:372 msg="writing series" size="636 kB"
level=warn ts=2026-04-29T20:54:38.110118575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:54:40.109756063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:54:40.11324228Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-29T20:54:40.860262336Z caller=checkpoint.go:372 msg="writing series" size="278 kB"
level=info ts=2026-04-29T20:54:42.379973231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=186402ce1bc8ee9d immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="898 kB" avg_uncomp="898 kB" full=1 labels="{stream=\"integration2-hlit\"}"
level=error ts=2026-04-29T20:54:42.383496885Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/186402ce1bc8ee9d/19ddac5b409%3A19ddac9e257%3Af9a45c27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-hlit\"}"
level=info ts=2026-04-29T20:54:42.413542473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.1 MB" avg_comp="1.0 MB" total_uncomp="16 MB" avg_uncomp="8.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-29T20:54:42.417294058Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-29T20:54:43.110675184Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-29T20:54:43.110696586Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:54:43.11071365Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:54:43.110722012Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-29T20:54:43.58375594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:54:43.586482799Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:54:43.801175807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="489 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:54:43.80849362Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:54:44.23584775Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=debug ts=2026-04-29T20:54:44.235871014Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
ts=2026-04-29T20:54:46.458625939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=info ts=2026-04-29T20:54:46.982822375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:54:46.986564983Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-29T20:54:47.610771266Z caller=checkpoint.go:372 msg="writing series" size="761 kB"
level=info ts=2026-04-29T20:54:47.899375294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-29T20:54:47.901969357Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19ddaad9c07%3A19ddac3cf9e%3Aed3cd4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-04-29T20:54:48.110632979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:54:48.993478805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:54:48.995945849Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-29T20:54:49.182704611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:54:49.190307674Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:54:49.190329885Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:54:49.190355131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:54:49.196880887Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-29T20:54:49.841731062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:54:49.848837827Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-29T20:54:50.985156713Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=debug ts=2026-04-29T20:54:53.015917163Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-29T20:54:53.109903818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:54:54.360075919Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-04-29T20:54:54.833716245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:54:54.8367231Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-29T20:54:55.898618267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:54:55.901603557Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-29T20:54:57.443915713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:54:57.449722517Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-29T20:54:57.736301925Z caller=checkpoint.go:372 msg="writing series" size="745 kB"
level=debug ts=2026-04-29T20:54:57.736334698Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=4
level=warn ts=2026-04-29T20:54:58.110393847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:54:58.870394121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:54:58.877907208Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:55:00.178388772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:55:00.181608895Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-29T20:55:00.679792234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:55:00.68228892Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-29T20:55:00.940482499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3498cb8c2f07200d immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-dvis\"}"
level=error ts=2026-04-29T20:55:00.943363702Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3498cb8c2f07200d/19ddab85df6%3A19ddacd801f%3Af49f9823?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-dvis\"}"
level=debug ts=2026-04-29T20:55:01.110369092Z caller=checkpoint.go:372 msg="writing series" size="141 kB"
level=info ts=2026-04-29T20:55:02.087972095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:55:02.094990834Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-29T20:55:02.135029118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:55:02.13789244Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-29T20:55:02.74203555Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-29T20:55:03.110382204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:55:04.485520049Z caller=checkpoint.go:372 msg="writing series" size="321 kB"
level=debug ts=2026-04-29T20:55:07.86070219Z caller=checkpoint.go:372 msg="writing series" size="653 B"
level=warn ts=2026-04-29T20:55:08.110689748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:55:08.372467422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:55:08.376137504Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-29T20:55:11.235258199Z caller=checkpoint.go:372 msg="writing series" size="742 kB"
level=debug ts=2026-04-29T20:55:11.235283489Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-04-29T20:55:11.907993241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:55:11.91217306Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:55:11.912193673Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:55:11.912216423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:55:11.915809702Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-29T20:55:13.087917072Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:55:13.088431Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:55:13.110533525Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:55:13.110563138Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:55:13.110571138Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:55:13.110633671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:55:14.61020696Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-04-29T20:55:14.717546782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=801f5e087572d469 immediate=false num_chunks=1 total_comp="307 kB" avg_comp="307 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"build-e2e-cwjf\"}"
level=error ts=2026-04-29T20:55:14.721106553Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/801f5e087572d469/19ddab80be5%3A19ddac62d16%3Aafbdc1db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-cwjf\"}"
ts=2026-04-29T20:55:16.46166553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-cjk6f-6f926783 10.128.0.79:7946"
level=info ts=2026-04-29T20:55:16.555493175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:55:16.563420221Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T20:55:17.477484931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5968eb640ba84b08 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"build-e2e-cmho\"}"
level=error ts=2026-04-29T20:55:17.480886741Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5968eb640ba84b08/19ddab8e2bd%3A19ddacd31f6%3Ae4c1710c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-cmho\"}"
level=debug ts=2026-04-29T20:55:17.985428263Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-04-29T20:55:18.110978713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:55:18.403499364Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:55:20.813293648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-29T20:55:20.816446762Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19ddaad73c7%3A19ddaae1c08%3A6ff787bd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-29T20:55:21.360166137Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-04-29T20:55:21.767187116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="447 kB" avg_comp="447 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-29T20:55:21.771079313Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19ddaad7881%3A19ddaf207a6%3A7b1886ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-04-29T20:55:23.110213953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:55:24.73532493Z caller=checkpoint.go:372 msg="writing series" size="9.3 kB"
level=info ts=2026-04-29T20:55:24.853863807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="962 B" avg_comp="962 B" total_uncomp="6.4 kB" avg_uncomp="6.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-29T20:55:24.85689355Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19ddaae7b65%3A19ddaae7f2a%3Aafcc1a1b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-29T20:55:25.034486303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:55:25.038561764Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-29T20:55:28.110320119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:55:28.121316434Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-04-29T20:55:28.121337972Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=5
level=info ts=2026-04-29T20:55:28.526001531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-29T20:55:28.530087932Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19ddaae71a6%3A19ddaae7efb%3A7c7de59f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-29T20:55:30.310713894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.1 MB" avg_comp="1.0 MB" total_uncomp="16 MB" avg_uncomp="8.1 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
ts=2026-04-29T20:55:30.313303207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.79:40738"
level=error ts=2026-04-29T20:55:30.314407358Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19ddaae1c06%3A19ddac2aab3%3A24bd4510?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-29T20:55:31.485651406Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-04-29T20:55:31.596986275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:55:31.599731998Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-04-29T20:55:33.110327592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:55:33.483735672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="489 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:55:33.489870228Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:55:33.489890099Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-29T20:55:33.48992455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="489 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:55:33.495415847Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:55:34.862876115Z caller=checkpoint.go:372 msg="writing series" size="3.0 MB"
level=debug ts=2026-04-29T20:55:34.862898489Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=2
level=warn ts=2026-04-29T20:55:38.110010495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:55:38.235355204Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-04-29T20:55:38.574194684Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-29T20:55:39.460600178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-29T20:55:39.464048408Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19ddaae1c06%3A19ddad6c6d2%3Ae5cbf217?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-29T20:55:39.907482721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=186402ce1bc8ee9d immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="898 kB" avg_uncomp="898 kB" full=1 labels="{stream=\"integration2-hlit\"}"
level=error ts=2026-04-29T20:55:39.910580774Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/186402ce1bc8ee9d/19ddac5b409%3A19ddac9e257%3Af9a45c27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-hlit\"}"
level=info ts=2026-04-29T20:55:40.370889862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-29T20:55:40.377573384Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19ddaad7c84%3A19ddaec7af5%3Acaaeafe2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-29T20:55:41.609994303Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=info ts=2026-04-29T20:55:42.89998381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-29T20:55:42.909357992Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19ddaad73d4%3A19ddaccecd6%3A338d6cd9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-29T20:55:43.042870759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="590 kB" avg_comp="590 kB" total_uncomp="6.4 MB" avg_uncomp="6.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-29T20:55:43.046886464Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19ddaad78db%3A19ddadc9c74%3Aa1a37c74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-29T20:55:43.109843106Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:55:43.109873634Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-29T20:55:43.109869569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:55:43.109882631Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-29T20:55:44.098429652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:55166"
level=debug ts=2026-04-29T20:55:44.985992006Z caller=checkpoint.go:372 msg="writing series" size="653 kB"
level=info ts=2026-04-29T20:55:45.894918867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-29T20:55:45.897790853Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19ddaad9c07%3A19ddaae79c9%3A20a0b5de?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-04-29T20:55:46.464664684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"
level=info ts=2026-04-29T20:55:46.756013065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="141 MB" avg_uncomp="24 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-29T20:55:46.762160263Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19ddaad7aab%3A19ddaae762a%3A2d216fe8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-29T20:55:47.207450823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-29T20:55:47.210524893Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19ddaad9c07%3A19ddac3cf9e%3Aed3cd4ea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-04-29T20:55:48.110834299Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:55:48.360428106Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=debug ts=2026-04-29T20:55:48.408464293Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:55:48.906080582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-29T20:55:48.908844518Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19ddaae79c8%3A19ddaae7b66%3A1d63cfd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-29T20:55:49.32314002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-29T20:55:49.325921727Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19ddaae1c07%3A19ddab1e1cf%3Ac6d14f04?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-29T20:55:50.351823598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.0 kB" avg_comp="9.0 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-29T20:55:50.354540843Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19ddaae71a6%3A19ddaae79c9%3A798aa7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-29T20:55:50.422728661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="489 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-29T20:55:50.429136858Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19ddaad7a3e%3A19ddab88781%3A8f401bc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-29T20:55:51.736018807Z caller=checkpoint.go:372 msg="writing series" size="362 kB"
level=debug ts=2026-04-29T20:55:51.736042626Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-04-29T20:55:52.50535704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3498cb8c2f07200d immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-dvis\"}"
level=error ts=2026-04-29T20:55:52.507925199Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3498cb8c2f07200d/19ddab85df6%3A19ddacd801f%3Af49f9823?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-dvis\"}"
level=info ts=2026-04-29T20:55:52.773489161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-29T20:55:52.776847967Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19ddaad78c4%3A19ddad5e99c%3A143dc2be?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-29T20:55:53.110167914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:55:55.110309707Z caller=checkpoint.go:372 msg="writing series" size="355 kB"
level=info ts=2026-04-29T20:55:55.974481617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-29T20:55:55.977577765Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19ddaae71a7%3A19ddaae79c9%3A5f16809c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-29T20:55:57.543027565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41772c565df34325 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-zciz\"}"
level=error ts=2026-04-29T20:55:57.549496542Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41772c565df34325/19ddab9a616%3A19ddacb73bd%3Ab4e4347b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-zciz\"}"
level=info ts=2026-04-29T20:55:57.899477573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-29T20:55:57.902782943Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19ddaad7c60%3A19ddae362d4%3A94b35122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-29T20:55:58.110100878Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:55:58.499252836Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-04-29T20:55:58.499274196Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=2
level=info ts=2026-04-29T20:56:01.548173225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-29T20:56:01.552957084Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19ddaad9c07%3A19ddaae7efb%3A9115068?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-29T20:56:01.860797659Z caller=checkpoint.go:372 msg="writing series" size="262 kB"
level=warn ts=2026-04-29T20:56:03.109900194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-29T20:56:04.521741725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5968eb640ba84b08 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"build-e2e-cmho\"}"
level=error ts=2026-04-29T20:56:04.524728669Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5968eb640ba84b08/19ddab8e2bd%3A19ddacd31f6%3Ae4c1710c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-cmho\"}"
level=debug ts=2026-04-29T20:56:05.235508819Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=info ts=2026-04-29T20:56:06.521108303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=801f5e087572d469 immediate=false num_chunks=1 total_comp="307 kB" avg_comp="307 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"build-e2e-cwjf\"}"
level=error ts=2026-04-29T20:56:06.524430489Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/801f5e087572d469/19ddab80be5%3A19ddac62d16%3Aafbdc1db?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-cwjf\"}"
level=warn ts=2026-04-29T20:56:08.110815223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:56:08.610349507Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=debug ts=2026-04-29T20:56:11.985771576Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-29T20:56:13.088280729Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-29T20:56:13.088866585Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.64:11211 count=1
level=info ts=2026-04-29T20:56:13.110253829Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-29T20:56:13.110281325Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-29T20:56:13.110292326Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-29T20:56:13.11031678Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-29T20:56:13.147754193Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-29T20:56:13.33023221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-29T20:56:13.335103209Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19ddab85df6%3A19ddab9e880%3A345e94bb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-29T20:56:15.360182215Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
ts=2026-04-29T20:56:16.467396391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-twmvd-1869990c 10.130.0.79:7946"